Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ffcea0cb07 | ||
|
|
a3f8314c9a | ||
|
|
d5ccb1091a | ||
|
|
528774d640 | ||
|
|
3a9ea9e330 | ||
|
|
25f2d9d0e4 | ||
|
|
5ab6778b97 | ||
|
|
26697472cf | ||
|
|
2abd4ffcb6 | ||
|
|
a9ac303d3e | ||
|
|
740602b9bf | ||
|
|
e31a588f19 | ||
|
|
cda626be10 | ||
|
|
b7ce1d467c | ||
|
|
52fb999af6 | ||
|
|
e0453fa07d | ||
|
|
2525a12264 | ||
|
|
8e09dd85f8 | ||
|
|
addb3d1b32 | ||
|
|
86884e368e | ||
|
|
51f881681b | ||
|
|
27483ae1d6 | ||
|
|
595c29e17c | ||
|
|
acf93a1f36 | ||
|
|
84278293df | ||
|
|
dc24d80ed4 | ||
|
|
3810af66da | ||
|
|
eb339195c7 | ||
|
|
5d1065c1bc | ||
|
|
6ffb5bd8c1 | ||
|
|
dddf79ec2f | ||
|
|
5ebb4cb506 | ||
|
|
0b1d905866 | ||
|
|
52c15adaf2 | ||
|
|
3372626e34 | ||
|
|
98bb841a6a | ||
|
|
98bf0a9d66 | ||
|
|
17bb14fb18 | ||
|
|
70f98b352f | ||
|
|
efa4aeeeda | ||
|
|
a7d41eb5f3 | ||
|
|
76eead976f | ||
|
|
922d589b09 | ||
|
|
a975ab627a | ||
|
|
9bedd96774 | ||
|
|
383c85231a | ||
|
|
de62035889 | ||
|
|
b456232d00 | ||
|
|
c9e95ecea6 | ||
|
|
7987bd8a37 | ||
|
|
3411265dc1 | ||
|
|
098b8dbe43 | ||
|
|
f48f16830e | ||
|
|
955c23d321 | ||
|
|
d3e1d8219c | ||
|
|
4480839db0 | ||
|
|
8e25567ff2 | ||
|
|
8f65bd184e | ||
|
|
ae9c248987 | ||
|
|
bd6aaa1e43 | ||
|
|
2c65edfffd | ||
|
|
6c2c01e67b | ||
|
|
ef343dbec0 | ||
|
|
a58a398427 | ||
|
|
d510c8a534 | ||
|
|
dc5206f9d3 | ||
|
|
89e65ca331 | ||
|
|
17916b411a | ||
|
|
3ef2c38eb4 | ||
|
|
4c63d6660d | ||
|
|
a453b77f04 | ||
|
|
8e1004fe6f | ||
|
|
663f30feeb | ||
|
|
6653d81a80 | ||
|
|
21df7c8348 | ||
|
|
97381e65d0 | ||
|
|
f8ced6d819 | ||
|
|
f254588409 | ||
|
|
3e7c79fa69 | ||
|
|
47c12f9405 | ||
|
|
bd0914b7ed | ||
|
|
079cee4e9c | ||
|
|
edf2f22ef9 | ||
|
|
e1ef56cdbc | ||
|
|
4d0abc61d5 | ||
|
|
7c2b22bafa | ||
|
|
5e5c8a42a9 | ||
|
|
2b3094ffc5 | ||
|
|
9c91eb5d22 | ||
|
|
2df60fbf07 | ||
|
|
a2d17d693c | ||
|
|
c4917619ee | ||
|
|
760bdae278 | ||
|
|
1b946113c6 | ||
|
|
b0e51c6f89 | ||
|
|
7455fff401 | ||
|
|
aaf1843b09 | ||
|
|
a4737b92e4 | ||
|
|
75ae810866 | ||
|
|
9d7526ee57 | ||
|
|
fa0b0edd63 | ||
|
|
bfc6d199c0 | ||
|
|
4d6e59b80a | ||
|
|
4646aae320 | ||
|
|
eddeca82d3 | ||
|
|
69edeecc45 | ||
|
|
6088e476a7 | ||
|
|
ce171fa954 | ||
|
|
113da76c36 | ||
|
|
4ef11901db | ||
|
|
29abf609f3 | ||
|
|
e13d93a3ed | ||
|
|
b9850b1710 | ||
|
|
905a9f2856 | ||
|
|
1d8b83c37c | ||
|
|
62dbf35ea3 | ||
|
|
e9ccd9bc98 | ||
|
|
cc8e0648db | ||
|
|
ff80cc96fd | ||
|
|
295c1db5f1 | ||
|
|
0521149cf1 | ||
|
|
b4a6569d0d | ||
|
|
1833f594cb | ||
|
|
1615838f33 | ||
|
|
7fa8f4ebde | ||
|
|
6354454cc9 | ||
|
|
c47c193ad1 | ||
|
|
db10bfbbca | ||
|
|
dac9d89f91 | ||
|
|
81936e9f56 | ||
|
|
86f467bd65 | ||
|
|
7d480e2ad0 | ||
|
|
ad136c3fe8 | ||
|
|
9c4a999e54 | ||
|
|
3f5634a44f | ||
|
|
f4bc60d659 | ||
|
|
7055548260 | ||
|
|
477271907e | ||
|
|
50923bb684 | ||
|
|
d2e40dcffb | ||
|
|
58878958d8 | ||
|
|
b47d22cf3b | ||
|
|
e804ff3b8e | ||
|
|
cd6f119883 | ||
|
|
321776480e | ||
|
|
7c0647f87a | ||
|
|
efd7b217ca | ||
|
|
3d7ef9e945 | ||
|
|
7fce7a7ce5 | ||
|
|
f5018bdcbc | ||
|
|
b82c22b888 | ||
|
|
f839a461e6 | ||
|
|
8525666913 | ||
|
|
3f24248e01 | ||
|
|
6b86673af4 | ||
|
|
d924c665f8 | ||
|
|
aa51644f5f | ||
|
|
378b7559e4 | ||
|
|
05121fbc99 | ||
|
|
bf518223e5 | ||
|
|
332d24a646 | ||
|
|
c5e022f056 | ||
|
|
4065ac7e6a | ||
|
|
ce08fc2aec | ||
|
|
ea6d829c29 | ||
|
|
6a7ff330f6 | ||
|
|
7e1aeeb9fe | ||
|
|
7d682c90d5 | ||
|
|
cc9fc3b544 | ||
|
|
c9101e3450 | ||
|
|
51daf3e388 | ||
|
|
09d351a0e7 | ||
|
|
b5078003ce | ||
|
|
f2c32cfb8e | ||
|
|
1453fa4a33 | ||
|
|
efe106dfda | ||
|
|
ec63056bd7 | ||
|
|
09becb3d00 | ||
|
|
8d2ec5e867 | ||
|
|
0e9839e73d | ||
|
|
05ab963951 | ||
|
|
8142484347 | ||
|
|
0d91b82f86 | ||
|
|
90fc86d437 | ||
|
|
2fe58073e4 | ||
|
|
bc03e82687 | ||
|
|
3f983e6d76 | ||
|
|
134ec9a68e | ||
|
|
9cf177b198 | ||
|
|
a5fae489e0 | ||
|
|
8d0083d4e4 | ||
|
|
9f177557d2 | ||
|
|
46fce4a767 | ||
|
|
2d9595b0e5 | ||
|
|
a7d7456279 | ||
|
|
44a792c34f | ||
|
|
3987cd4260 | ||
|
|
95d9ed4aa6 | ||
|
|
8d0063d514 | ||
|
|
0ca0cac534 | ||
|
|
cf0c55ba80 | ||
|
|
fa4f823fc8 | ||
|
|
884060891e | ||
|
|
955c8c3445 | ||
|
|
99df44d7fb | ||
|
|
30fbdc52b2 | ||
|
|
046ff735cf | ||
|
|
7afaba0547 | ||
|
|
db8ed5a004 | ||
|
|
0ef25a8037 | ||
|
|
17a26894b0 | ||
|
|
946c8f26db | ||
|
|
8759b5860d | ||
|
|
dc902d2c0e | ||
|
|
cedcbb1434 | ||
|
|
4f72b193aa | ||
|
|
7b94e9a651 | ||
|
|
c6bf84a578 | ||
|
|
444f9b8166 | ||
|
|
7a47223a58 | ||
|
|
bc138e3b37 | ||
|
|
df7c3db085 | ||
|
|
6fda49369f | ||
|
|
8018cd4685 | ||
|
|
29875b7367 | ||
|
|
7ae5c57b6b | ||
|
|
dc4e8b9fcd | ||
|
|
6f9ba428d5 | ||
|
|
c60359bfed | ||
|
|
e6b76e156a | ||
|
|
e74986cc9a | ||
|
|
2df45b27d3 | ||
|
|
bacd6aec4f | ||
|
|
fad301e7a4 |
@@ -1,22 +1,50 @@
|
||||
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
||||
# independent of any one machine. Two parallel jobs match the two local lanes:
|
||||
# independent of any one machine. Two jobs match the two local lanes:
|
||||
# frontend — tsc (3 projects) + eslint + prettier --check + knip + vitest
|
||||
# backend — ruff + the full pytest suite (pure-logic + the DB-backed lane) against a real Postgres
|
||||
#
|
||||
# They are declared side by side but they DO NOT run in parallel: the lab has a single act_runner
|
||||
# with one slot, so they queue. Measured across runs #86-#91 — the backend job starts at the exact
|
||||
# second the frontend one ends, every time, and the pair costs ~4 + ~6 = **~10 minutes from the
|
||||
# push**. Anything that waits on this gate has to budget for the sum, not the max; the ops
|
||||
# `release` read-back timed out on its first real run for believing otherwise.
|
||||
#
|
||||
# Runner: the lab act_runner (LXC 115) mounts the Docker socket, so service containers work. The backend
|
||||
# runs pytest NATIVELY against a `postgres` service (not the local docker-compose.test.yml, whose
|
||||
# bind-mount is unreliable under a mounted-socket runner); fidelity is kept by installing the SAME pinned
|
||||
# requirements, Postgres 16, and ffmpeg, and by setting SIFTLODE_TEST_REQUIRE_DB=1 so a DB/ffmpeg that
|
||||
# failed to come up is a hard error, never a silent skip. node 22 matches the prod build image
|
||||
# (Dockerfile `node:22-alpine`).
|
||||
# failed to come up is a hard error, never a silent skip.
|
||||
#
|
||||
# The frontend container's node major is NOT free: it must equal the prod build image's and the dev
|
||||
# machine's, because vitest/tsc read the engine's own Intl and web-API implementations and those split
|
||||
# between majors. `.nvmrc` is the single source of truth and
|
||||
# `frontend/src/lib/nodeEngine.contract.test.ts` fails this gate if any of them drift apart.
|
||||
name: gate
|
||||
|
||||
on: [push, pull_request]
|
||||
# Branch pushes only. A release pushes `main` and then the `v<ver>` tag, and a bare `on: [push]`
|
||||
# treated the tag as a second push of the IDENTICAL commit — every release ran this whole gate
|
||||
# twice, to the same verdict, for nothing (runs #81-#90 are five such pairs). `branches: ['**']`
|
||||
# keeps every branch covered and drops tag refs.
|
||||
#
|
||||
# It has to be `branches`, NOT the `tags-ignore` this obviously wants to be. Gitea's matchPushEvent
|
||||
# requires EVERY named filter to match, and each one bails on the wrong ref kind: `tags-ignore`
|
||||
# starts with `if !refName.IsTag() { break }`, so a filter meant to skip tags rejects every BRANCH
|
||||
# push and the gate stops running entirely. `branches` bails the same way on a tag, which is the
|
||||
# direction we want. (Measured in go-gitea v1.27.3 modules/actions/workflows.go, not assumed — it
|
||||
# differs from GitHub, where `tags-ignore` alone leaves branches unfiltered.)
|
||||
#
|
||||
# If that filter were ever wrong the gate would not run at all — which is why `siftlode release`
|
||||
# now ends by reading the verdict back for the sha it pushed, and says so out loud when there is
|
||||
# no run for it. A gate that silently stops running is the failure this repo already lived through.
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
container: node:22-bookworm
|
||||
container: node:24-bookworm
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
@@ -34,7 +62,11 @@ jobs:
|
||||
# A combined python+node image: python 3.12 (matches the prod Dockerfile) for the tests, and node
|
||||
# so the JS actions (checkout) run. actions/setup-python is NOT usable here — the runner can't reach
|
||||
# GitHub's python-versions download, so it must be baked into the image.
|
||||
container: nikolaik/python-nodejs:python3.12-nodejs20
|
||||
#
|
||||
# This node runs the checkout action, never our code, so it is deliberately OUTSIDE the engine
|
||||
# contract the frontend job is held to — it tracks `.nvmrc` as hygiene (node 20 went EOL in April
|
||||
# 2026), but a missing upstream tag here must never be able to block the gate.
|
||||
container: nikolaik/python-nodejs:python3.12-nodejs24
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
@@ -61,6 +93,22 @@ jobs:
|
||||
run: python -m ruff check --config backend/ruff.toml backend scripts
|
||||
- name: ruff format --check
|
||||
run: python -m ruff format --check --config backend/ruff.toml backend scripts
|
||||
# The ninth lane (E27 S3). Run from backend/ so it resolves exactly as the local lane does —
|
||||
# pyrightconfig.json's include/extraPaths are relative to the config, but the baseline is
|
||||
# found relative to the working directory. New errors fail; the ~200 pre-existing ones are in
|
||||
# .basedpyright/baseline.json and burn down epic by epic. Matching is per (file, rule, column
|
||||
# span) plus a count, so a new error of the SAME shape in a file where a sibling was just
|
||||
# fixed is absorbed — burn the hot files down first.
|
||||
#
|
||||
# The second step is not redundant. basedpyright TIGHTENS the baseline in place when the count
|
||||
# drops, with no --writebaseline, and CI throws its workspace away — so without this a
|
||||
# burn-down is green here and red on the developer's `siftlode check`, which has the same
|
||||
# guard. Two gates over one tree must not answer differently.
|
||||
- name: basedpyright
|
||||
run: python -m basedpyright --project pyrightconfig.json
|
||||
working-directory: backend
|
||||
- name: basedpyright baseline is committed
|
||||
run: git diff --exit-code -- backend/.basedpyright/baseline.json
|
||||
- name: pytest
|
||||
run: python -m pytest
|
||||
working-directory: backend
|
||||
|
||||
@@ -37,3 +37,7 @@ backups/
|
||||
Thumbs.db
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# TypeScript incremental build artifacts — machine-local, absolute-path-bearing, and no gate lane
|
||||
# reads or regenerates them, so a committed one is stale from the commit that adds it.
|
||||
*.tsbuildinfo
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
3.12
|
||||
+839
-1319
File diff suppressed because it is too large
Load Diff
@@ -7,17 +7,18 @@ worse than no rule — if you change how the code works, change this file in the
|
||||
|
||||
Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a download center). It is a
|
||||
**modular monolith**: one Postgres, one FastAPI process (uvicorn + APScheduler), one worker process
|
||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 18 + react-query frontend with a router-free navigation
|
||||
provider.
|
||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 19 (with the React Compiler) + react-query frontend, with a
|
||||
router-free navigation provider.
|
||||
|
||||
## The gate — never merge or ship red
|
||||
|
||||
The local gate is **ruff (`check` + `format --check`) + typecheck (3 tsc projects) + eslint + prettier
|
||||
The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + typecheck (3 tsc projects) + eslint + prettier
|
||||
--check + knip + vitest + pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
||||
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
||||
|
||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` + `ruff format --check backend` and `pytest` (the DB-lane needs Postgres).
|
||||
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` and burn down epic by epic. Fixed some? Regenerate with `--writebaseline`.
|
||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||
|
||||
## Workflow
|
||||
@@ -45,20 +46,22 @@ how it gets built. Concretely:
|
||||
behind each option and the weight of the decision. These decision points are where the product owner
|
||||
engages with the build and learns from it, so make them understandable — a reasoned menu, not a bare one.
|
||||
- **Mockups for design/UX decisions.** For any visual or interaction choice, show mockup options to
|
||||
choose from before building.
|
||||
choose from before building. How the UI looks and behaves is decided in
|
||||
[`docs/DESIGN.md`](docs/DESIGN.md) (its variants render in the dev-only `/design-lab`): build to it,
|
||||
and change it only through its decision log.
|
||||
- **Challenge unsound requests.** If a request is technically ill-advised and you can back it with real
|
||||
arguments, push back or propose an alternative — don't comply blindly.
|
||||
- **Verify UI in the browser before calling it done.** tsc/unit/DOM tests miss visual and interaction
|
||||
bugs; do a real visual check of any UI change.
|
||||
bugs; do a real visual check of any UI change, and run DESIGN.md's four-step Conformance check.
|
||||
- **Be terse everywhere else.** Outside those decision points, lead with the answer or decision. During
|
||||
implementation and debugging especially: no restating the same point reworded, no filler.
|
||||
|
||||
## Architecture shape (don't fight it)
|
||||
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`,
|
||||
`security`, `logs`, `logsink`, `diagnostics`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `bodylimit`, `progress`,
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||
its menus, `Modal`/`Overlay`, `Button`/`form`, `DataTable`/`VirtualGrid`, `QueryState`, …). That
|
||||
@@ -74,29 +77,37 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
|
||||
## Canonical patterns — when you have a choice, pick THIS
|
||||
|
||||
| Area | Use | Not |
|
||||
| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------ |
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`) | ad-hoc `{"unsubscribed"}`/`{"cleared": n}`/echo |
|
||||
| Transaction commit | the handler/job-wrapper commits; helpers `flush` | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE action button | `ui/Button` — pick a `SIZES` rung | a hand-rolled `<button className="px-3 py-2 …">` |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE control | the chip/segmented primitives (`Select` only for >6 options) | a native `<select>` |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
| Area | Use | Not |
|
||||
| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Request SIZE | nothing — `bodylimit` caps every route (SEC-20); need more? add a ceiling to `ROUTE_MAX_BODY_BYTES` | `capped_body` on a route that does not read raw bytes itself |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE button | `ui/Button` (a `SIZES` rung) · `IconButton` · `Pressable` (a clickable card/row) · `Chip` · `DragHandle` | a raw `<button>` or `FocusSafeButton` outside the control primitives (lint-enforced) |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE text field / checkbox / slider | `ui/Input` · `Textarea` · `InnerInput` in a `fieldClass` frame · `Checkbox`/`CheckRow` · `Slider` | a raw `<input>`/`<textarea>` anywhere but those primitives (lint-enforced) |
|
||||
| FE glass surface | a role: `SURFACE.panel` / `card` / `menu` / `popover` / `tooltip` (`ui/surfaces.ts`) | a bare `glass`/`glass-card`/`glass-menu` beside a hand-picked radius (lint-enforced) |
|
||||
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
||||
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
||||
| FE effect that must not re-subscribe | `useEffectEvent` (React 19.2+), else a latest-value ref | an excused dep list asserting the same thing in prose |
|
||||
| FE hook suppression | an inline `eslint-disable-next-line` NAMING the invariant | `eslint-suppressions.json` (deliberately absent — E27 S2) |
|
||||
|
||||
## House rules that bite (learned from real bugs)
|
||||
|
||||
|
||||
+3
-1
@@ -10,7 +10,9 @@ ARG GIT_SHA=unknown
|
||||
ARG BUILD_DATE=
|
||||
|
||||
# Stage 1: build the frontend SPA
|
||||
FROM node:22-alpine AS frontend
|
||||
# The major is pinned to `.nvmrc` and held there by frontend/src/lib/nodeEngine.contract.test.ts:
|
||||
# this image, the CI gate's image and the dev machine must be ONE engine (see that test for why).
|
||||
FROM node:24-alpine AS frontend
|
||||
WORKDIR /fe
|
||||
# Deps first, keyed on the lockfile, so editing app source never re-resolves node_modules.
|
||||
COPY frontend/package.json frontend/package-lock.json ./
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,81 @@
|
||||
"""add debug_sessions + debug_screenshots, and index log_events.debug_session_id (E16)
|
||||
|
||||
Revision ID: 0089_debug_sessions
|
||||
Revises: 0088_channels_feed_fallback_at
|
||||
Create Date: 2026-09-19
|
||||
|
||||
A debug session is one recording: Start → reproduce → Stop (app/debugsession.py). Every log line
|
||||
of that user's requests while it runs carries its id in `log_events.debug_session_id` — the column
|
||||
E22 created and left NULL — and the issue it ends in is built from exactly those rows, so the column
|
||||
gets its index now. Partial: nearly every row has no session, and an index of NULLs would be most
|
||||
of the table for nothing.
|
||||
|
||||
At most one OPEN session per user, held by a partial unique index rather than by the start handler
|
||||
alone: two tabs pressing Start at once would otherwise both pass a "none open" check.
|
||||
|
||||
Screenshots are rows, not files: bounded (size and count per session, app/debugsession.py), gone
|
||||
with their session by cascade, and a self-host has no file volume to configure for them.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0089_debug_sessions"
|
||||
down_revision: str | None = "0088_channels_feed_fallback_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"debug_sessions",
|
||||
sa.Column("id", sa.String(16), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer,
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("level", sa.String(8), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("until", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("ended_reason", sa.String(16), nullable=True),
|
||||
sa.Column("issue_url", sa.String(512), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_debug_sessions_open_per_user",
|
||||
"debug_sessions",
|
||||
["user_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("ended_at IS NULL"),
|
||||
)
|
||||
op.create_table(
|
||||
"debug_screenshots",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column(
|
||||
"session_id",
|
||||
sa.String(16),
|
||||
sa.ForeignKey("debug_sessions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("mime", sa.String(16), nullable=False),
|
||||
sa.Column("size", sa.Integer, nullable=False),
|
||||
sa.Column("data", sa.LargeBinary, nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_log_events_debug_session",
|
||||
"log_events",
|
||||
["debug_session_id"],
|
||||
postgresql_where=sa.text("debug_session_id IS NOT NULL"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_log_events_debug_session", table_name="log_events")
|
||||
op.drop_table("debug_screenshots")
|
||||
op.drop_index("uq_debug_sessions_open_per_user", table_name="debug_sessions")
|
||||
op.drop_table("debug_sessions")
|
||||
@@ -0,0 +1,27 @@
|
||||
"""add debug_sessions.draft — the issue being written from a session (E16 S3)
|
||||
|
||||
Revision ID: 0090_debug_session_draft
|
||||
Revises: 0089_debug_sessions
|
||||
Create Date: 2026-09-19
|
||||
|
||||
The composer saves as it goes: the title, the rendered markdown body that an export or an issue
|
||||
carries, the log rows the user took out, and the form's own fields so reopening it later (Settings →
|
||||
Diagnosztika lists past sessions) shows what was written. NULL until the composer first saves.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
revision: str = "0090_debug_session_draft"
|
||||
down_revision: str | None = "0089_debug_sessions"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("debug_sessions", sa.Column("draft", JSONB, nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("debug_sessions", "draft")
|
||||
@@ -13,16 +13,17 @@ Three layers, by *who* runs the code — not by feature:
|
||||
response models, status-code mapping, serialization. It owns logic that runs *only* on a request.
|
||||
Business logic that a request happens to trigger does **not** live here if anything else runs it too.
|
||||
|
||||
2. **`plex/`, `downloads/`, `sync/` — domain logic shared by more than one caller.** Anything the
|
||||
2. **`plex/`, `downloads/`, `sync/`, `issues/` — domain logic shared by more than one caller.** Anything the
|
||||
**scheduler or the worker** runs (not just a request) lives in one of these domain packages, so the
|
||||
route, the worker, and the scheduler share one definition. Examples from this codebase:
|
||||
`downloads/service.py` owns enqueue + the job state transitions (routes call it); `plex/sync.py` is
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline.
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline;
|
||||
`issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker.
|
||||
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table) and `diagnostics` (a user's consented log
|
||||
level), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||
level) and `debugsession` (a recording that tags its user's lines, E16), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
module must never import a `routes/…` module).
|
||||
|
||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||
|
||||
@@ -58,6 +58,9 @@ class AuditAction:
|
||||
DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete"
|
||||
# The log itself was cleared (a single row survives the clear as tamper-evidence)
|
||||
AUDIT_CLEAR = "audit_clear"
|
||||
# E16: a user sent a debug session's issue to the tracker — the one act that takes their log
|
||||
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
|
||||
ISSUE_CREATE = "issue_create"
|
||||
|
||||
|
||||
def record(
|
||||
|
||||
+12
-10
@@ -4,7 +4,7 @@ from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
from app import diagnostics
|
||||
from app import debugsession
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
@@ -14,6 +14,7 @@ from app.db import get_db
|
||||
from app.models import Invite, LogEvent, User
|
||||
from app.security import decrypt_optional
|
||||
from app.utils import canonical_email
|
||||
from app.refusal import refused
|
||||
|
||||
PASSWORD_MIN_LEN = 10
|
||||
|
||||
@@ -341,19 +342,19 @@ def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
|
||||
def _session_user(request: Request, db: Session, user_id: int | None, is_default: bool) -> User:
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
raise refused(401, "not_authenticated")
|
||||
user = db.get(User, user_id)
|
||||
if user is None or not user.is_active or user.is_suspended:
|
||||
# Suspended/deactivated mid-session → end it so the block takes effect at once.
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
raise refused(401, "not_authenticated")
|
||||
# SA4: reject a cookie whose recorded epoch is behind the account's current one (revoked by a
|
||||
# password reset/change or "log out everywhere"). A cookie with no recorded epoch (pre-SA4, or an
|
||||
# account signed in before this shipped) is treated as epoch 0 — so the FIRST bump revokes it too.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(user_id), 0)) != (user.session_epoch or 0):
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
raise refused(401, "not_authenticated")
|
||||
# Always keep the session default in the switchable list — covers sessions created before
|
||||
# multi-session existed (their account_ids was never seeded), so the switcher isn't empty.
|
||||
default_id = request.session.get("user_id")
|
||||
@@ -366,7 +367,11 @@ def _session_user(request: Request, db: Session, user_id: int | None, is_default
|
||||
# session middleware and this task's context. `request.state` is shared across middleware layers
|
||||
# (it lives on the ASGI scope), so the access-log line can read it back. The user's consented
|
||||
# diagnostics level rides along: their requests are logged — and stored — in that detail (S5).
|
||||
logs.set_user(user.id, diagnostics.effective_level(user), user.diagnostics_until)
|
||||
# So does a debug session the request names (E16): only the caller's own open one counts, and a
|
||||
# header naming anything else is ignored rather than refused — a stale tab must keep working.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, until = debugsession.level_for(user, session)
|
||||
logs.set_user(user.id, level, until, session.id if session else None)
|
||||
request.state.user_id = user.id
|
||||
return user
|
||||
|
||||
@@ -418,10 +423,7 @@ def require_write_scope(user: User = Depends(require_human)) -> User:
|
||||
checks that had drifted into three different 403 messages across channels/playlists. Layers on
|
||||
`require_human` because writing to YouTube needs a real, non-demo identity."""
|
||||
if not has_write_scope(user):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Enable YouTube editing in Settings first.",
|
||||
)
|
||||
raise refused(403, "youtube_editing_off")
|
||||
return user
|
||||
|
||||
|
||||
@@ -429,7 +431,7 @@ def admin_user(user: User = Depends(current_user)) -> User:
|
||||
"""Dependency: require the signed-in user to be an admin. Single source of truth for the
|
||||
admin gate — every admin route/action depends on this rather than re-checking the role."""
|
||||
if user.role != "admin":
|
||||
raise HTTPException(status_code=403, detail="Admin only")
|
||||
raise refused(403, "admin_only")
|
||||
return user
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""The global request-body cap (SEC-20).
|
||||
|
||||
**The hole.** FastAPI reads a declared body IN FULL before auth or any other dependency runs — the
|
||||
body parameter has to be parsed to be passed in. So `POST /auth/password-login` drains a 32 MiB
|
||||
upload from an anonymous caller and only then answers 401. 70 of the 72 body-declaring routes behave
|
||||
the same way: one unauthenticated client can spend the process's memory and time at will.
|
||||
|
||||
**Why a middleware and not `capped_body` on every route.** `routes/_common.py:capped_body` is the
|
||||
right tool where a route genuinely takes bulk bytes, and the two routes that use it use it
|
||||
correctly. But putting it on all 70 others would (a) fight the canonical "Request body → a Pydantic
|
||||
model" row, since capping means taking the body as raw bytes and parsing it by hand, and (b) leave
|
||||
the NEXT route added uncovered, because nothing would remind its author. A default that has to be
|
||||
opted into is not a default. This module inverts it: the cap applies to everything, and the few
|
||||
routes that need more opt OUT, in one list, next to the code that reads it.
|
||||
|
||||
**Why raw ASGI and not `@app.middleware("http")`.** Starlette's BaseHTTPMiddleware cannot wrap
|
||||
`receive`, so it can only read `Content-Length` — which a client may omit entirely (chunked) or
|
||||
simply understate. The bytes have to be counted as they arrive, which is the same rule `capped_body`
|
||||
already applies per-route, and counting needs the receive channel.
|
||||
|
||||
**Why the refusal is byte-identical to `capped_body`'s.** Two shapes for one fact is how the
|
||||
`cancelled`/`canceled` bug happened. The 413 body here is built through the same `JSONResponse`
|
||||
FastAPI renders `HTTPException(413, {"code": "too_large"})` with — not a hand-rolled `json.dumps`,
|
||||
whose default separators differ by two bytes — and `tests/test_body_limit.py` compares the RAW BYTES
|
||||
of the two against each other rather than against a literal, so they cannot drift apart. The same
|
||||
reasoning decides where the middleware is registered (`main.py`): just inside CORS, so this 413
|
||||
carries the same headers `capped_body`'s does.
|
||||
|
||||
**No admin knob, deliberately.** The ceiling is a constant, not a `sysconfig` value: an attacker-
|
||||
facing limit that an admin can raise from the UI is a limit that can be raised by whoever gets the
|
||||
admin session, and nobody has asked to tune it. If a self-hoster ever needs a different ceiling it
|
||||
belongs in `config.py` (env, set before the process starts), not in the database.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from app import debugsession
|
||||
from app.config import settings
|
||||
|
||||
# 1 MiB. The largest body any route on the default ceiling legitimately takes is 512 KiB (the client
|
||||
# log sink, `routes/diagnostics.py:BODY_MAX_BYTES`), so this leaves 2x headroom — a ceiling that
|
||||
# refuses a legitimate request would be a worse bug than the one this module fixes. That ratio is
|
||||
# the load-bearing fact, so `test_body_limit.py` holds it rather than the figure above.
|
||||
DEFAULT_MAX_BODY_BYTES = 1024 * 1024
|
||||
|
||||
# The routes that need MORE than the default, as (method, path TEMPLATE) → their own ceiling. The
|
||||
# template, not a concrete path, so the list reads like the route decorators it mirrors and
|
||||
# `test_body_limit.py` can assert that each one still resolves to a real route: a renamed route
|
||||
# fails the gate instead of silently losing its allowance in prod.
|
||||
#
|
||||
# These are RAISED CEILINGS, NOT EXEMPTIONS, and the difference is the whole point. An exemption
|
||||
# would mean "this route is counted by something else" — which is only true when the route's own
|
||||
# limit applies to the bytes as they ARRIVE. That, in turn, is only true of a route with NO DECLARED
|
||||
# BODY PARAMETER: FastAPI parses and spools a declared body BEFORE the first dependency runs, so a
|
||||
# route that declares one has already stored the bytes by the time its own cap — or its auth — gets
|
||||
# a say.
|
||||
#
|
||||
# Both listed routes now satisfy that. The screenshot route always did (`capped_body` streams it,
|
||||
# and the owner check resolves first). The upload route did NOT until P1: it declared
|
||||
# `file: UploadFile = File(...)`, so an anonymous caller could make the process write the whole
|
||||
# per-file allowance to temp before `require_human` refused them — this middleware bounded that
|
||||
# number but could not stop the writing. It now reads its own body from a dependency declared after
|
||||
# the auth one, so its in-handler cap measures bytes as they arrive, like everything else here.
|
||||
#
|
||||
# That rule is no longer only written here: `test_body_limit.py` parses the routes and fails the
|
||||
# gate if any route in this dict declares a body parameter. A comment could not have stopped the
|
||||
# defect it describes; the test can.
|
||||
#
|
||||
# A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why
|
||||
# `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the
|
||||
# default, so listing it could only widen what it accepts.
|
||||
ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
||||
# Ingesting a local file. The body IS the file since P1 — no multipart envelope to allow for —
|
||||
# so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s
|
||||
# `download_upload_max_bytes` is bounded by this same constant (see sysconfig.SPECS), so an
|
||||
# admin can only ever LOWER the route's cap below the edge, never raise it past one.
|
||||
#
|
||||
# WHICH LAYER ANSWERS, measured rather than assumed (deep-review round 1):
|
||||
# * knob LOWERED (an override below the default) — the route's cap is the smaller number, so
|
||||
# `capped_stream_to` trips first and answers 422 `upload_too_large` carrying the admin's own
|
||||
# figure, after draining the rest of the body so the client can read it.
|
||||
# * knob at the DEFAULT — the two numbers are EQUAL, and the edge always wins: a declared
|
||||
# Content-Length over the cap is refused here before a byte is read, and an undeclared body
|
||||
# trips this counter on the same chunk the route would have tripped on. The route's 422 is
|
||||
# unreachable in that configuration, and no amount of headroom would change it — the
|
||||
# declared-length check fires first for any body big enough to matter, and the route's drain
|
||||
# pushes the count past the edge for the rest.
|
||||
#
|
||||
# So this 413 is a refusal a real user meets on a default install, not a theoretical edge, and
|
||||
# it has to be as readable as the route's own: the client localizes `too_large` (`errors.tooLarge`)
|
||||
# exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before
|
||||
# sending it at all, off the `upload_max_bytes` that `GET /api/downloads/usage` reports.
|
||||
("POST", "/api/downloads/upload"): settings.download_upload_max_bytes,
|
||||
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
|
||||
}
|
||||
|
||||
# The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders
|
||||
# as — built through the same JSONResponse so the separators match too, and asserted against a real
|
||||
# route's raw bytes in `test_body_limit.py` rather than against a literal here.
|
||||
TOO_LARGE_BODY = JSONResponse({"detail": {"code": "too_large"}}, status_code=413).body
|
||||
|
||||
_PATH_PARAM = re.compile(r"\{[^/{}]+\}")
|
||||
|
||||
|
||||
def template_to_regex(template: str) -> re.Pattern[str]:
|
||||
"""`/api/debug-sessions/{sid}/screenshots` → a regex matching one concrete path.
|
||||
|
||||
A path parameter matches `[^/]+` — one segment, never across a `/`, so `{sid}` cannot swallow a
|
||||
deeper path and hand an exemption to a route that was never granted one.
|
||||
"""
|
||||
literals = [re.escape(part) for part in _PATH_PARAM.split(template)]
|
||||
return re.compile("^" + "[^/]+".join(literals) + "$")
|
||||
|
||||
|
||||
def _declared_length(scope: Scope) -> int | None:
|
||||
"""The `Content-Length` header as an int, or None when absent or unparseable (a chunked body has
|
||||
none, and a malformed one is not a number we may trust — both fall through to counting)."""
|
||||
for name, value in scope.get("headers", ()):
|
||||
if name == b"content-length":
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
# silent-ok: a malformed Content-Length is the caller's doing, not a fault of ours,
|
||||
# and it is about to be answered anyway — returning None drops the request into the
|
||||
# counting path, which refuses on the real bytes. Logging it would hand an anonymous
|
||||
# client a free line in our log for every request it cares to malform.
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
async def _send_too_large(send: Send) -> None:
|
||||
await send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": 413,
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"content-length", str(len(TOO_LARGE_BODY)).encode()),
|
||||
],
|
||||
}
|
||||
)
|
||||
await send({"type": "http.response.body", "body": TOO_LARGE_BODY})
|
||||
|
||||
|
||||
class BodyLimitMiddleware:
|
||||
"""Refuses any request body over its route's ceiling, before the application sees it.
|
||||
|
||||
Registered just inside CORS in `main.py` — so the 413 is decorated like every other response
|
||||
and a browser can actually read it — but outside the session, the setup gate and every auth
|
||||
dependency, none of which run for a request that is already too large.
|
||||
|
||||
Every request gets a finite ceiling: `ROUTE_MAX_BODY_BYTES` raises it for the two routes that
|
||||
genuinely take bulk bytes, and there is no path that skips counting altogether.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app: ASGIApp,
|
||||
*,
|
||||
limit: int = DEFAULT_MAX_BODY_BYTES,
|
||||
routes: dict[tuple[str, str], int] | None = None,
|
||||
) -> None:
|
||||
self.app = app
|
||||
self.limit = limit
|
||||
self._routes = tuple(
|
||||
(method, template_to_regex(path), cap)
|
||||
for (method, path), cap in (ROUTE_MAX_BODY_BYTES if routes is None else routes).items()
|
||||
)
|
||||
|
||||
def limit_for(self, method: str, path: str) -> int:
|
||||
"""This request's ceiling: the route's own if it has one, else the global default. Every
|
||||
request gets a finite number — there is no path through here that is simply not counted."""
|
||||
for m, rx, cap in self._routes:
|
||||
if m == method and rx.match(path):
|
||||
return cap
|
||||
return self.limit
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
limit = self.limit_for(scope.get("method", ""), scope.get("path", ""))
|
||||
|
||||
declared = _declared_length(scope)
|
||||
if declared is not None and declared > limit:
|
||||
# Refused on the declaration alone: nothing downstream is called, and not one byte of
|
||||
# the body is read.
|
||||
await _send_too_large(send)
|
||||
return
|
||||
|
||||
seen = 0
|
||||
refused = False
|
||||
started = False
|
||||
|
||||
async def counting_receive() -> Message:
|
||||
nonlocal seen, refused
|
||||
message = await receive()
|
||||
if message["type"] != "http.request":
|
||||
return message
|
||||
seen += len(message.get("body", b""))
|
||||
if seen > limit:
|
||||
# The app is blocked on this receive, so it has not answered yet. Telling it the
|
||||
# client went away is the only in-band way to stop it reading: `request.stream()`
|
||||
# turns an `http.disconnect` into ClientDisconnect, which unwinds the handler. Our
|
||||
# own 413 goes out below, once the app has finished unwinding.
|
||||
refused = True
|
||||
return {"type": "http.disconnect"}
|
||||
return message
|
||||
|
||||
async def guarded_send(message: Message) -> None:
|
||||
nonlocal started
|
||||
if refused and not started:
|
||||
return # discard the app's answer to the disconnect; the 413 below is the response
|
||||
started = True
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, counting_receive, guarded_send)
|
||||
except Exception: # noqa: BLE001 — only swallowed when WE caused it, see below
|
||||
# ClientDisconnect (or whatever a handler raises on a truncated read) is the expected
|
||||
# consequence of the disconnect we just faked, and the caller must get the 413 rather
|
||||
# than a 500. Anything raised when we did NOT refuse is a real failure and is re-raised
|
||||
# untouched.
|
||||
if not refused:
|
||||
raise
|
||||
|
||||
if refused and not started:
|
||||
await _send_too_large(send)
|
||||
+17
-2
@@ -1,7 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args, get_origin
|
||||
|
||||
from pydantic import ValidationInfo, field_validator, model_validator
|
||||
from pydantic import Field, ValidationInfo, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Shipped placeholder; production must override it. Used by the startup guard below.
|
||||
@@ -247,7 +247,11 @@ class Settings(BaseSettings):
|
||||
# Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker
|
||||
# finishes), an upload's size is known at ingest, so this bounds a single file directly; the
|
||||
# user's overall byte footprint is still governed by download_default_max_bytes.
|
||||
download_upload_max_bytes: int = 524_288_000 # 500 MiB
|
||||
# ge=1: this number is also the EDGE ceiling (`bodylimit.ROUTE_MAX_BODY_BYTES`), where 0 is a
|
||||
# real limit of zero rather than the route's "no per-file cap" sentinel — an instance set to 0
|
||||
# would refuse every upload, including the admin page's own way back. To lift the cap in
|
||||
# practice, set a large number.
|
||||
download_upload_max_bytes: int = Field(default=524_288_000, ge=1) # 500 MiB
|
||||
# Retention: an asset expires this many days after its last access; the GC job then
|
||||
# deletes it (once no job references it) after an additional grace window.
|
||||
download_retention_days: int = 30
|
||||
@@ -315,6 +319,10 @@ class Settings(BaseSettings):
|
||||
# re-encode is CPU-heavy (the VAAPI hardware path is far cheaper and could sustain more); direct-serve
|
||||
# and remux (stream-copy) are not capped by this.
|
||||
plex_max_transcodes: int = 4
|
||||
# Max concurrent playback sessions of ANY kind — remux (stream-copy) included. Cheap on the CPU but
|
||||
# each one is a whole segmented copy of the film on the scratch volume, written faster than
|
||||
# realtime. One account holds at most two regardless (plex/stream.py `_MAX_SESSIONS_PER_USER`).
|
||||
plex_max_sessions: int = 8
|
||||
# --- Transcode (playable="transcode") re-encode tuning (E6 S1) ---
|
||||
# Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the
|
||||
# source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render
|
||||
@@ -347,6 +355,13 @@ class Settings(BaseSettings):
|
||||
tmdb_api_base: str = "https://api.themoviedb.org/3"
|
||||
# Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default.
|
||||
tmdb_language: str = "en-US"
|
||||
|
||||
# --- Bug reports (E16): where a debug session's issue goes. All three empty = no tracker; the
|
||||
# composer still exports a zip. The repository must be PRIVATE — an issue carries the reporter's
|
||||
# log — and app/issues/gitea.py refuses a public one. Runtime knobs (sysconfig, admin UI). ---
|
||||
issues_gitea_url: str = ""
|
||||
issues_gitea_repo: str = ""
|
||||
issues_gitea_token: str = ""
|
||||
# How often the TMDB enrichment job fetches a batch of not-yet-enriched Plex titles, in minutes.
|
||||
tmdb_enrich_interval_min: int = 120
|
||||
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||
|
||||
+51
-2
@@ -1,6 +1,6 @@
|
||||
from collections.abc import Iterator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
from app.config import settings
|
||||
@@ -12,7 +12,9 @@ class Base(DeclarativeBase):
|
||||
|
||||
# Pool sized above the sync-endpoint threadpool (~40) so a burst of short DB sessions can't starve on
|
||||
# checkout. The image proxies deliberately DON'T hold a connection during their slow external fetch
|
||||
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones.
|
||||
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones. The one
|
||||
# long hold — a playlist sync's advisory claim, for a whole push — opens its own unpooled connection
|
||||
# (sync/playlists.py `_claim_engine`) instead of drawing a third one from here.
|
||||
engine = create_engine(
|
||||
settings.database_url,
|
||||
pool_pre_ping=True,
|
||||
@@ -34,3 +36,50 @@ def get_db() -> Iterator[Session]:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def release_read_connection(db: Session) -> None:
|
||||
"""End a request's READ transaction so its pooled connection goes back BEFORE a long await.
|
||||
|
||||
SQLAlchemy 2.0 autobegins on the first SELECT, so a request that has merely authenticated is
|
||||
already holding one of the connections above — `idle in transaction` to Postgres, and nothing
|
||||
times it out. A route that then awaits something slow (a request body still arriving, an
|
||||
external fetch) pins it for the whole wait, which is precisely the invariant the pool is sized
|
||||
on in the comment above. Measured: the next checkout after this call reuses the very backend
|
||||
that was released, and a 1.8 s upload goes from one sustained `idle in transaction` to none.
|
||||
|
||||
Every route that reads its own body from a dependency needs this, and `test_body_limit.py`
|
||||
holds that — the three that do are the ones with a raised (or self-imposed) body ceiling, and
|
||||
they got here one at a time, which is how the upload route ended up the only one fixed.
|
||||
|
||||
The caller must have NOTHING PENDING, and that is checked rather than described. The session is
|
||||
shared with every dependency that ran before, so a rollback here would silently discard a write
|
||||
one of them staged — `service.enqueue` already learned that lesson the expensive way and uses a
|
||||
SAVEPOINT for it. CLAUDE.md: a rule stated in a comment must also be enforced in code.
|
||||
|
||||
TWO checks, because the obvious one is blind to the shapes this codebase actually writes in.
|
||||
`db.new`/`deleted`/`dirty` hold only UNFLUSHED unit-of-work: the moment a helper calls
|
||||
`db.flush()` — CLAUDE.md's canonical split is "the handler commits; helpers flush" — the object
|
||||
is persistent with no unflushed history and all three sets are empty again, and Core DML
|
||||
(`db.execute(update(...))`) never enters them at all. Measured: after `add`+`flush` the sets are
|
||||
empty, the old check passed, and the rollback lost the row. So the transaction is asked
|
||||
directly. Postgres assigns a transaction id on the first WRITE and never for a read, so
|
||||
`pg_current_xact_id_if_assigned()` is NULL for exactly the state this function is safe in — and
|
||||
it sees a raw `text("UPDATE …")` too, which no session-level bookkeeping would.
|
||||
"""
|
||||
if db.new or db.deleted or db.dirty:
|
||||
raise RuntimeError(
|
||||
"release_read_connection() with pending work: something ahead of this dependency "
|
||||
"staged a write, which ending the transaction would discard. Commit it first, or do "
|
||||
"it after the await."
|
||||
)
|
||||
if (
|
||||
db.in_transaction()
|
||||
and db.execute(text("SELECT pg_current_xact_id_if_assigned()")).scalar() is not None
|
||||
):
|
||||
raise RuntimeError(
|
||||
"release_read_connection() on a transaction that has already WRITTEN: something ahead "
|
||||
"of this dependency flushed a row or ran an UPDATE/INSERT/DELETE, which ending the "
|
||||
"transaction would discard. Commit it first, or do it after the await."
|
||||
)
|
||||
db.rollback()
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, and the evidence a bug report is built from.
|
||||
|
||||
While a session is open its user's browser names it in `X-Debug-Session` on every request. The auth
|
||||
dependency accepts the header only for the caller's OWN open session (anything else is ignored, never
|
||||
an error — a stale tab must not break the app) and then logs that request at the session's level with
|
||||
every line tagged by its id; the browser tags its own events the same way. The issue is later built
|
||||
from exactly the rows carrying the tag, so what the review list shows is what gets sent.
|
||||
|
||||
Starting a session IS the consent for its level, for its duration: the start dialog carries the
|
||||
consent text. It needs no change to the user's own diagnostics level, and none is made — when the
|
||||
session ends the level in force is simply the user's own again.
|
||||
|
||||
A session is capped at `MAX_MINUTES`, set by the server at Start. An open one past its cap is over
|
||||
whether or not anything has closed it yet: it reads as over at once, a new Start closes it, and the
|
||||
hourly log job sweeps the rest. Sessions go with the logs they point at, after `log_retention_days`.
|
||||
|
||||
Core layer: imports `models`, `diagnostics`.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import StrEnum
|
||||
from typing import Literal, get_args
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import and_, delete, func, not_, select, update
|
||||
from sqlalchemy.orm import Session, defer
|
||||
|
||||
from app import diagnostics
|
||||
from app.models import DebugScreenshot, DebugSession, LogEvent, User
|
||||
|
||||
HEADER = "x-debug-session"
|
||||
MAX_MINUTES = diagnostics.SESSION_MINUTES
|
||||
SCREENSHOT_MAX_BYTES = 10 * 1024 * 1024
|
||||
SCREENSHOTS_MAX = 8
|
||||
# What one user may keep in screenshots across ALL their sessions (until retention takes them): the
|
||||
# per-session cap alone would let Start → 8 images → Stop repeat without end.
|
||||
SCREENSHOT_USER_MAX_BYTES = 100 * 1024 * 1024
|
||||
# The screenshot types kept, and the extension each is exported and sent under — one table, so an
|
||||
# accepted type always has a name.
|
||||
IMAGE_TYPES = {"image/png": "png", "image/jpeg": "jpg", "image/webp": "webp"}
|
||||
# What a screenshot is called in the zip and on the issue. `issues/bundle.py` builds the names from
|
||||
# this and `IMAGE_TYPES`; this module and `issues/gitea.py` read them back to tell a screenshot's
|
||||
# link from a log's. One definition, because three copies of a name is how the next rename breaks
|
||||
# `images_pending` into silence.
|
||||
SCREENSHOT_PREFIX = "screenshot-"
|
||||
# Rows an issue can carry. A 60-minute session at Teljes is a few thousand rows; past this the
|
||||
# bundle says how many it left out rather than growing without bound.
|
||||
BUNDLE_MAX = 5_000
|
||||
|
||||
SessionLevel = Literal["DEBUG", "TRACE"]
|
||||
SESSION_LEVELS: tuple[str, ...] = get_args(SessionLevel)
|
||||
# Detail order of the diagnostics levels: a session raises the level in force, never lowers it.
|
||||
_DETAIL = {"INFO": 0, "DEBUG": 1, "TRACE": 2}
|
||||
_ID_LEN = 16
|
||||
|
||||
|
||||
class EndReason(StrEnum):
|
||||
STOPPED = "stopped"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
def is_live(session: DebugSession, now: datetime | None = None) -> bool:
|
||||
"""Open and within its cap — THE liveness rule: an open session past its cap is over."""
|
||||
return session.ended_at is None and session.until > (now or datetime.now(timezone.utc))
|
||||
|
||||
|
||||
def open_session(db: Session, user: User) -> DebugSession | None:
|
||||
"""The user's live session, or None. Read-only: one past its cap is over and reads as None;
|
||||
`start` and the hourly sweep are what close it."""
|
||||
session = _unclosed(db, user)
|
||||
if session is None or not is_live(session, datetime.now(timezone.utc)):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def _unclosed(db: Session, user: User) -> DebugSession | None:
|
||||
return db.scalar(
|
||||
select(DebugSession).where(DebugSession.user_id == user.id, DebugSession.ended_at.is_(None))
|
||||
)
|
||||
|
||||
|
||||
def start(db: Session, user: User, level: SessionLevel) -> DebugSession:
|
||||
"""Open a session, or return the live one — a second tab pressing Start joins the recording
|
||||
rather than ending it. One past its cap is closed first: the one-open-per-user index would
|
||||
refuse the new row beside it. Flushes; the caller commits. Two concurrent Starts can still both
|
||||
get here: the index refuses the second, and the caller joins the one that won."""
|
||||
now = datetime.now(timezone.utc)
|
||||
current = _unclosed(db, user)
|
||||
if current is not None:
|
||||
if is_live(current, now):
|
||||
return current
|
||||
end(current, EndReason.EXPIRED)
|
||||
db.flush()
|
||||
session = DebugSession(
|
||||
id=secrets.token_hex(_ID_LEN // 2),
|
||||
user_id=user.id,
|
||||
level=level,
|
||||
started_at=now,
|
||||
until=now + timedelta(minutes=MAX_MINUTES),
|
||||
)
|
||||
db.add(session)
|
||||
db.flush()
|
||||
return session
|
||||
|
||||
|
||||
def stop(session: DebugSession) -> None:
|
||||
"""The user's Stop. Past its cap the session had already ended on its own, and a late Stop must
|
||||
not rename that."""
|
||||
end(session, EndReason.STOPPED if is_live(session) else EndReason.EXPIRED)
|
||||
|
||||
|
||||
def end(session: DebugSession, reason: EndReason) -> None:
|
||||
"""Close a session. An expired one is dated at its cap, not at whenever it was noticed."""
|
||||
if session.ended_at is not None:
|
||||
return
|
||||
session.ended_at = (
|
||||
session.until
|
||||
if reason is EndReason.EXPIRED
|
||||
else min(datetime.now(timezone.utc), session.until)
|
||||
)
|
||||
session.ended_reason = reason
|
||||
|
||||
|
||||
def from_header(db: Session, user: User, value: str | None) -> DebugSession | None:
|
||||
"""The session a request names, if it is the caller's own and still live; else None."""
|
||||
if not value or len(value) != _ID_LEN or not all(c in "0123456789abcdef" for c in value):
|
||||
return None
|
||||
session = db.get(DebugSession, value)
|
||||
if (
|
||||
session is None
|
||||
or session.user_id != user.id
|
||||
or not is_live(session, datetime.now(timezone.utc))
|
||||
):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def level_for(user: User, session: DebugSession | None) -> tuple[str, datetime | None]:
|
||||
"""The diagnostics level (and its expiry) a request is logged at: the user's own, raised to the
|
||||
session's while one is named — whichever records more.
|
||||
|
||||
A named session also caps the expiry at its own: this pair is what the process tracks under the
|
||||
session (app/logs.py), and an expiry outliving the recording would hold the root logger down
|
||||
after it ended."""
|
||||
own = diagnostics.effective_level(user)
|
||||
until = user.diagnostics_until if own != diagnostics.BASE else None
|
||||
if session is None:
|
||||
return own, until
|
||||
if _DETAIL[session.level] <= _DETAIL[own]:
|
||||
return own, min(until, session.until) if until else session.until
|
||||
return session.level, session.until
|
||||
|
||||
|
||||
def lock(db: Session, session: DebugSession) -> None:
|
||||
"""Re-read the session under a row lock, held until the caller commits. Everything that changes
|
||||
what an issue is made of — the draft, the screenshots, the send itself — takes it first: a send
|
||||
in flight sees none of them change under it, and each of them sees a send that committed."""
|
||||
db.refresh(session, with_for_update=True)
|
||||
|
||||
|
||||
def save_draft(session: DebugSession, draft: dict) -> None:
|
||||
"""Keep the composer's draft (the caller validated, locked and commits). Shape: `title`, `body`
|
||||
(the rendered markdown an export or an issue carries), `excluded` (log row ids taken out of the
|
||||
evidence) and `form` (the composer's own fields, to reopen it as it was). A delivery's progress
|
||||
is not the composer's to replace, and is kept."""
|
||||
progress = (session.draft or {}).get("delivery")
|
||||
session.draft = {**draft, "delivery": progress} if progress else draft
|
||||
|
||||
|
||||
# --- a delivery's progress -----------------------------------------------------------------
|
||||
# Written by app/issues/gitea.py into the draft; read HERE, so the send's decision and what the
|
||||
# composer shows are one reading of it.
|
||||
def delivery(session: DebugSession) -> dict:
|
||||
return dict((session.draft or {}).get("delivery") or {})
|
||||
|
||||
|
||||
def not_uploaded(session: DebugSession) -> list[str]:
|
||||
"""The files a send that stopped half-way has still to upload — by name, as the issue carries
|
||||
them. What the composer names to the user."""
|
||||
return list(delivery(session).get("pending") or [])
|
||||
|
||||
|
||||
def images_pending(session: DebugSession) -> bool:
|
||||
"""Whether the issue's body still has to be edited to show the screenshots it already carries
|
||||
(every upload went, the edit that shows them did not)."""
|
||||
state = delivery(session)
|
||||
shown = state.get("images_shown")
|
||||
return not shown and any(n.startswith(SCREENSHOT_PREFIX) for n in state.get("links") or {})
|
||||
|
||||
|
||||
def unfinished(session: DebugSession) -> bool:
|
||||
"""Whether a send owes the issue anything — the retry's question."""
|
||||
return bool(not_uploaded(session)) or images_pending(session)
|
||||
|
||||
|
||||
# A send claims the session while it talks to the tracker, instead of holding the row's lock across
|
||||
# those calls: a Stop, an autosave or a screenshot would queue behind a lock for the whole send.
|
||||
# They see the claim and are refused at once instead. A claim not renewed for this long was left by
|
||||
# a request that died mid-send (the process restarted), and the next send takes it over — which is
|
||||
# why a live send RENEWS it every time it reports progress (`set_delivery`). A claim is a lease, and
|
||||
# a send slower than the lease must not lose it while it is still running.
|
||||
SEND_CLAIM_SECONDS = 600
|
||||
|
||||
|
||||
def _claim(session: DebugSession) -> dict:
|
||||
claim = delivery(session).get("sending")
|
||||
return claim if isinstance(claim, dict) else {}
|
||||
|
||||
|
||||
def _stamp(token: str) -> dict:
|
||||
return {"at": datetime.now(timezone.utc).isoformat(), "by": token}
|
||||
|
||||
|
||||
def sending(session: DebugSession) -> bool:
|
||||
"""Whether a send is under way right now (a live claim)."""
|
||||
at = _claim(session).get("at")
|
||||
if not at:
|
||||
return False
|
||||
return (datetime.now(timezone.utc) - datetime.fromisoformat(at)).total_seconds() < (
|
||||
SEND_CLAIM_SECONDS
|
||||
)
|
||||
|
||||
|
||||
def claim_send(session: DebugSession) -> str:
|
||||
"""Mark the session as being sent, and return the token that OWNS this claim. The caller holds
|
||||
the lock and commits, which is what makes the claim visible to everyone else."""
|
||||
token = uuid4().hex
|
||||
_write_delivery(session, {**delivery(session), "sending": _stamp(token)})
|
||||
return token
|
||||
|
||||
|
||||
def release_send(session: DebugSession, token: str) -> None:
|
||||
"""Give back our OWN claim, and only ours: one that aged out and was taken over belongs to the
|
||||
send holding it now, and handing that one back would let a third send in beside it."""
|
||||
if _claim(session).get("by") != token:
|
||||
return
|
||||
_write_delivery(session, {k: v for k, v in delivery(session).items() if k != "sending"})
|
||||
|
||||
|
||||
class SendClaimLost(Exception):
|
||||
"""This send's claim is no longer on the row: it aged out and another send took it over. The
|
||||
send must stop — what it uploads from here it can no longer record, so a retry would send it
|
||||
again — and it must not write, because the row now holds the other send's progress."""
|
||||
|
||||
|
||||
def claim_owner(db: Session, session: DebugSession) -> str | None:
|
||||
"""Who holds the row's claim RIGHT NOW, read from the ROW and not from the object.
|
||||
|
||||
`expire_on_commit` is off (app/db.py), so a checkpoint does not re-read: a send that stalled
|
||||
past its lease still carries its own dead token in memory long after another send took over.
|
||||
A targeted scalar, not `lock`/`db.refresh` — a refresh here would discard the `issue_url` that
|
||||
`gitea.deliver` assigns one line before its first progress report, and that link is the one
|
||||
thing the send must not lose."""
|
||||
return db.scalar(
|
||||
select(DebugSession.draft["delivery"]["sending"]["by"].as_string()).where(
|
||||
DebugSession.id == session.id
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def set_delivery(db: Session, session: DebugSession, state: dict, token: str) -> None:
|
||||
"""Write a delivery's progress (app/issues/gitea.py keeps it; this module reads it), and renew
|
||||
the claim while doing it — a send still reporting progress is still alive, so one slower than
|
||||
the lease cannot expire while it is still running.
|
||||
|
||||
The claim is not the caller's to carry: `gitea.deliver` threads ONE snapshot of this dict
|
||||
through a whole send, so a progress report that carried the claim wrote a stale one back at
|
||||
every step. It is stamped from `token` here instead, and only after the ROW confirms the claim
|
||||
is still ours. Without that check the write is a full-column overwrite with no WHERE: a send
|
||||
that lost its lease came back and stomped the take-over send's committed links, then put its
|
||||
own dead token back on an idle session (deep-review round 4). `release_send` has always made
|
||||
the same check; this is the other half of it."""
|
||||
if claim_owner(db, session) != token:
|
||||
raise SendClaimLost(session.id)
|
||||
state = {k: v for k, v in state.items() if k != "sending"}
|
||||
state["sending"] = _stamp(token)
|
||||
_write_delivery(session, state)
|
||||
|
||||
|
||||
def _write_delivery(session: DebugSession, state: dict) -> None:
|
||||
# A new dict, so the JSONB column sees the change.
|
||||
session.draft = {**(session.draft or {}), "delivery": state}
|
||||
|
||||
|
||||
def serialize(session: DebugSession | None, screenshots: int = 0) -> dict | None:
|
||||
if session is None:
|
||||
return None
|
||||
return {
|
||||
"id": session.id,
|
||||
"level": session.level,
|
||||
"started_at": session.started_at.isoformat(),
|
||||
"until": session.until.isoformat(),
|
||||
"ended_at": session.ended_at.isoformat() if session.ended_at else None,
|
||||
"ended_reason": session.ended_reason,
|
||||
"issue_url": session.issue_url,
|
||||
# A send that stopped half-way: the files it has still to upload, and whether the issue's
|
||||
# body has still to be edited to show the screenshots it already carries. Both are true of a
|
||||
# send that is RUNNING too — every step commits its progress — so `sending` goes with them:
|
||||
# without it a reload or a second tab renders a live send as a failed one, and offers a
|
||||
# retry that is certain to be refused.
|
||||
"pending_files": not_uploaded(session),
|
||||
"images_pending": images_pending(session),
|
||||
"sending": sending(session),
|
||||
"screenshots": screenshots,
|
||||
"draft": session.draft,
|
||||
}
|
||||
|
||||
|
||||
def view(db: Session, session: DebugSession) -> dict:
|
||||
"""The session as the API serves it — one shape wherever it is served."""
|
||||
return serialize(session, screenshot_count(db, session)) or {}
|
||||
|
||||
|
||||
def screenshot_count(db: Session, session: DebugSession) -> int:
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.count())
|
||||
.select_from(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
# --- screenshots ---------------------------------------------------------------------------
|
||||
# By content, not by the declared type: what is stored is served back to a browser as an image.
|
||||
_SIGNATURES = (
|
||||
(b"\x89PNG\r\n\x1a\n", "image/png"),
|
||||
(b"\xff\xd8\xff", "image/jpeg"),
|
||||
)
|
||||
|
||||
|
||||
def sniff(data: bytes) -> str | None:
|
||||
"""The image type `data` really is — one of IMAGE_TYPES — or None."""
|
||||
for magic, mime in _SIGNATURES:
|
||||
if data.startswith(magic):
|
||||
return mime
|
||||
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
return None
|
||||
|
||||
|
||||
def lock_user(db: Session, user_id: int) -> None:
|
||||
"""Hold this user's own lock for the rest of the transaction. A session's row lock bounds what
|
||||
THAT session may keep; the byte cap below is the user's across all of them, and a lock on one
|
||||
session cannot serialize a check that reads them all — two uploads to two sessions both read the
|
||||
total before either commits, and both pass."""
|
||||
db.execute(select(func.pg_advisory_xact_lock(user_id)))
|
||||
|
||||
|
||||
def stored_screenshot_bytes(db: Session, user_id: int) -> int:
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.sum(DebugScreenshot.size), 0))
|
||||
.join(DebugSession, DebugSession.id == DebugScreenshot.session_id)
|
||||
.where(DebugSession.user_id == user_id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
class ScreenshotRefused(Exception):
|
||||
"""A screenshot that cannot be kept; `code` is the refusal's machine word."""
|
||||
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def add_screenshot(db: Session, session: DebugSession, data: bytes) -> DebugScreenshot:
|
||||
"""Keep one screenshot. Locks the user and the session, and flushes; the caller commits.
|
||||
Refused while a send is under way or once the issue was sent (it can no longer carry it), past
|
||||
the per-session count or the user's storage, or when it is not an image we can show."""
|
||||
if not data:
|
||||
raise ScreenshotRefused("empty")
|
||||
mime = sniff(data)
|
||||
if mime is None:
|
||||
raise ScreenshotRefused("not_an_image")
|
||||
# Under both locks: two uploads at once cannot both pass the count (the session's), nor both
|
||||
# pass the user's byte cap from two different sessions (the user's). Always in this order.
|
||||
lock_user(db, session.user_id)
|
||||
lock(db, session)
|
||||
if sending(session):
|
||||
raise ScreenshotRefused("sending")
|
||||
if session.issue_url is not None:
|
||||
raise ScreenshotRefused("already_sent")
|
||||
if screenshot_count(db, session) >= SCREENSHOTS_MAX:
|
||||
raise ScreenshotRefused("too_many")
|
||||
if stored_screenshot_bytes(db, session.user_id) + len(data) > SCREENSHOT_USER_MAX_BYTES:
|
||||
raise ScreenshotRefused("storage_full")
|
||||
shot = DebugScreenshot(
|
||||
session_id=session.id,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
mime=mime,
|
||||
size=len(data),
|
||||
data=data,
|
||||
)
|
||||
db.add(shot)
|
||||
db.flush()
|
||||
return shot
|
||||
|
||||
|
||||
def remove_screenshot(db: Session, session: DebugSession, shot: DebugScreenshot) -> None:
|
||||
"""Delete one screenshot, under the lock. Refused while a send is under way or unfinished: a
|
||||
delivery names the screenshots by their order, and removing one would renumber what it still
|
||||
has to send. Once the issue has everything, this only drops OUR copy — the issue keeps its
|
||||
own. The caller commits."""
|
||||
lock(db, session)
|
||||
if sending(session):
|
||||
raise ScreenshotRefused("sending")
|
||||
if unfinished(session):
|
||||
raise ScreenshotRefused("already_sent")
|
||||
db.delete(shot)
|
||||
db.flush()
|
||||
|
||||
|
||||
def screenshots(db: Session, session: DebugSession) -> list[DebugScreenshot]:
|
||||
"""The session's screenshots, oldest first, WITHOUT their bytes loaded."""
|
||||
return list(
|
||||
db.scalars(
|
||||
select(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
.options(defer(DebugScreenshot.data))
|
||||
.order_by(DebugScreenshot.id)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --- the evidence --------------------------------------------------------------------------
|
||||
# The browser's own log upload (`POST` here, every few seconds while recording) is how the evidence
|
||||
# arrives, not part of it: its server lines would fill the bundle with the recording's plumbing. The
|
||||
# browser leaves its copies out for the same reason (frontend lib/diagnostics `isOwnRequest`).
|
||||
INGEST_PATH = "/api/diagnostics/events"
|
||||
|
||||
|
||||
def _evidence(session: DebugSession):
|
||||
upload_line = and_(
|
||||
LogEvent.event == "http.request", LogEvent.attrs["path"].astext == INGEST_PATH
|
||||
)
|
||||
# `IS TRUE`, not the bare condition: for a row with no event or no attrs it is NULL, and
|
||||
# `NOT NULL` would drop exactly the rows that are evidence.
|
||||
return (LogEvent.debug_session_id == session.id, not_(upload_line.is_(True)))
|
||||
|
||||
|
||||
def bundle(
|
||||
db: Session, session: DebugSession, *, drop_excluded: bool = False
|
||||
) -> tuple[list[LogEvent], int]:
|
||||
"""The session's log rows, browser and server interleaved in time, capped at BUNDLE_MAX — and
|
||||
how many rows it has in all, so a cut bundle can say so. `drop_excluded` leaves out the rows
|
||||
the user took out in the composer: what an export or an issue carries."""
|
||||
where = list(_evidence(session))
|
||||
excluded = (session.draft or {}).get("excluded") or [] if drop_excluded else []
|
||||
if excluded:
|
||||
where.append(LogEvent.id.not_in(excluded))
|
||||
rows = list(
|
||||
db.scalars(
|
||||
select(LogEvent).where(*where).order_by(LogEvent.ts, LogEvent.id).limit(BUNDLE_MAX)
|
||||
)
|
||||
)
|
||||
total = len(rows)
|
||||
if total == BUNDLE_MAX:
|
||||
total = db.scalar(select(func.count()).select_from(LogEvent).where(*where)) or total
|
||||
return rows, total
|
||||
|
||||
|
||||
# --- housekeeping (the hourly log job) -----------------------------------------------------
|
||||
def sweep(db: Session) -> int:
|
||||
"""Stage closing every open session past its cap; returns how many."""
|
||||
now = datetime.now(timezone.utc)
|
||||
result = db.execute(
|
||||
update(DebugSession)
|
||||
.where(DebugSession.ended_at.is_(None), DebugSession.until <= now)
|
||||
.values(ended_at=DebugSession.until, ended_reason=EndReason.EXPIRED.value)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Stage deleting sessions (and by cascade their screenshots) started before the log retention
|
||||
window — the rows they would bundle are gone by then too. Returns how many."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, retention_days))
|
||||
result = db.execute(delete(DebugSession).where(DebugSession.started_at < cutoff))
|
||||
return result.rowcount or 0
|
||||
@@ -27,6 +27,9 @@ DiagnosticsLevel = Literal["INFO", "DEBUG", "TRACE"]
|
||||
LEVELS: tuple[str, ...] = get_args(DiagnosticsLevel)
|
||||
BASE: DiagnosticsLevel = "INFO"
|
||||
ELEVATION_HOURS = 24
|
||||
# How long a debug session records at most (E16, app/debugsession.py). Here, beside the other
|
||||
# consent window, because the start dialog's consent text states it and reads it from `state`.
|
||||
SESSION_MINUTES = 60
|
||||
|
||||
|
||||
def effective_level(user: User, now: datetime | None = None) -> DiagnosticsLevel:
|
||||
@@ -47,6 +50,7 @@ def state(db: Session, user: User) -> dict:
|
||||
"level": level,
|
||||
"until": user.diagnostics_until.isoformat() if level != BASE else None,
|
||||
"retention_days": sysconfig.get_int(db, "log_retention_days"),
|
||||
"session_minutes": SESSION_MINUTES,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -81,7 +81,11 @@ def pick(
|
||||
return None
|
||||
|
||||
def _comfort() -> dict | None:
|
||||
for card in pins.playables(pins.shelf(db, user) if shelf is None else shelf):
|
||||
if shelf is None:
|
||||
own = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
||||
else:
|
||||
own = shelf
|
||||
for card in pins.playables(own):
|
||||
if cards.key(card) not in skip and fits(card["remaining_seconds"], budget):
|
||||
return card
|
||||
return None
|
||||
|
||||
+107
-15
@@ -8,14 +8,16 @@ second one.
|
||||
|
||||
Every shelf card carries a resolved `next`: the thing that actually plays when you click the tile —
|
||||
a show's in-progress-or-next episode, a channel's newest unwatched video, a playlist's first
|
||||
unwatched item. It costs a query per non-leaf pin, which is affordable precisely because this shelf is
|
||||
short and hand-curated, and it buys the property the whole surface exists for: one click, no
|
||||
intermediate page, no second decision.
|
||||
unwatched item. That buys the property the whole surface exists for: one click, no intermediate page,
|
||||
no second decision. Channels and playlists are resolved for the whole shelf at once (`_Batch`), so the
|
||||
cost does not grow a query or two per pin.
|
||||
|
||||
Domain module, so nothing here raises HTTP. `resolve_ref` returns None for "no such thing, or not
|
||||
yours" and the route turns that into its 404 — core never imports routes.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -73,6 +75,44 @@ def resolve_ref(db: Session, user: User, kind: str, ref: str) -> tuple[str, obje
|
||||
return (col, pl.id) if pl is not None and pl.user_id == user.id else None
|
||||
|
||||
|
||||
def reorder(db: Session, user: User, ids: list[int]) -> None:
|
||||
"""Put the listed pins in the listed order, WITHIN the slots they already hold.
|
||||
|
||||
The client sends the order it can SEE, which is not always the whole shelf: with Plex off the Home
|
||||
rail hides the Plex pins, and a stale tab may not know a pin added elsewhere. Numbering only the
|
||||
listed ids 0..n-1 collided with every pin left out — two tiles on one position, and the hidden
|
||||
ones jumped about when they came back. So the pins left out keep their slots, and the listed ones
|
||||
are dealt into the slots the listed set occupies. Ids that are not this user's, or repeated, are
|
||||
ignored rather than failing a stale replay wholesale. Flushes; the route commits."""
|
||||
# Locked: the rows it leaves alone are part of the answer too. Two overlapping reorders each
|
||||
# skipped the rows that already matched in their own snapshot and interleaved into two pins on
|
||||
# one slot; the second now waits and deals from the first one's result. Locked in id order and
|
||||
# sorted HERE: with `ORDER BY position … FOR UPDATE` Postgres sorts before it locks, so the one
|
||||
# that waited got the fresh rows in its stale order and dealt the pins it left out back to their
|
||||
# old slots — and one global lock order means two reorders cannot deadlock.
|
||||
locked = db.execute(
|
||||
select(ComfortPin)
|
||||
.where(ComfortPin.user_id == user.id)
|
||||
.order_by(ComfortPin.id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalars()
|
||||
current = sorted(locked, key=lambda p: (p.position, p.id))
|
||||
owned = {p.id for p in current}
|
||||
listed: list[int] = []
|
||||
for i in ids:
|
||||
if i in owned and i not in listed:
|
||||
listed.append(i)
|
||||
chosen = set(listed)
|
||||
queue = iter(listed)
|
||||
order = [next(queue) if p.id in chosen else p.id for p in current]
|
||||
by_id = {p.id: p for p in current}
|
||||
for position, pin_id in enumerate(order):
|
||||
if by_id[pin_id].position != position:
|
||||
by_id[pin_id].position = position
|
||||
db.flush()
|
||||
|
||||
|
||||
def list_pins(db: Session, user: User) -> list[ComfortPin]:
|
||||
return list(
|
||||
db.execute(
|
||||
@@ -103,7 +143,34 @@ def next_position(db: Session, user: User) -> int:
|
||||
)
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
@dataclass
|
||||
class _Batch:
|
||||
"""What the shelf resolves for every pinned channel and playlist in one go."""
|
||||
|
||||
channels: dict[str, Channel] = field(default_factory=dict)
|
||||
channel_summary: dict[str, tuple[int, dict | None]] = field(default_factory=dict)
|
||||
playlists: dict[int, Playlist] = field(default_factory=dict)
|
||||
playlist_summary: dict[int, tuple[int, dict | None]] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def load(cls, db: Session, user: User, pins: list[ComfortPin]) -> "_Batch":
|
||||
ch_ids = [p.channel_id for p in pins if p.channel_id is not None]
|
||||
pl_ids = [p.playlist_id for p in pins if p.playlist_id is not None]
|
||||
b = cls()
|
||||
if ch_ids:
|
||||
b.channels = {
|
||||
c.id: c for c in db.scalars(select(Channel).where(Channel.id.in_(ch_ids)))
|
||||
}
|
||||
b.channel_summary = queries.channels_summary(db, user, ch_ids)
|
||||
if pl_ids:
|
||||
b.playlists = {
|
||||
p.id: p for p in db.scalars(select(Playlist).where(Playlist.id.in_(pl_ids)))
|
||||
}
|
||||
b.playlist_summary = queries.playlists_summary(db, user, pl_ids)
|
||||
return b
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict | None:
|
||||
"""One pin as a unified card with its resolved `next`. None when the pinned row has gone (the FK
|
||||
cascade removes the pin itself, so this is only the window between a delete and a refetch)."""
|
||||
card: dict | None = None
|
||||
@@ -117,11 +184,11 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
nxt = dict(card) # a video plays itself
|
||||
|
||||
elif pin.channel_id is not None:
|
||||
ch = db.get(Channel, pin.channel_id)
|
||||
ch = batch.channels.get(pin.channel_id)
|
||||
if ch is None:
|
||||
return None
|
||||
card = cards.from_channel(ch, queries.channel_unwatched_count(db, user, ch.id))
|
||||
row = queries.channel_next_video(db, user, ch.id)
|
||||
count, row = batch.channel_summary.get(ch.id, (0, None))
|
||||
card = cards.from_channel(ch, count)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
elif pin.plex_item_id is not None:
|
||||
@@ -142,10 +209,10 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
nxt = cards.from_plex(ep) if ep else None
|
||||
|
||||
elif pin.playlist_id is not None:
|
||||
pl = db.get(Playlist, pin.playlist_id)
|
||||
pl = batch.playlists.get(pin.playlist_id)
|
||||
if pl is None:
|
||||
return None
|
||||
count, row = queries.playlist_summary(db, user, pl.id)
|
||||
count, row = batch.playlist_summary.get(pl.id, (0, None))
|
||||
card = cards.from_playlist(pl, count, row.get("thumbnail_url") if row else None)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
@@ -159,11 +226,24 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
return card
|
||||
|
||||
|
||||
def shelf(db: Session, user: User) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order."""
|
||||
_PLEX_COLUMNS = ("plex_item_id", "plex_show_id")
|
||||
|
||||
|
||||
def shelf(db: Session, user: User, *, include_plex: bool = True) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order.
|
||||
|
||||
`include_plex=False` is the Home surface with the deployment's Plex toggle off (the owner's call,
|
||||
2026-09-19): the rails beside it go YouTube-only, and a Plex tile there would offer a Play that
|
||||
lands on a gated module. The pins themselves are kept — the pin menus still read the full shelf
|
||||
(`GET /pins`), and switching Plex back on brings the tiles back in their old slots
|
||||
(`reorder` keeps a hidden pin's place)."""
|
||||
out = []
|
||||
for pin in list_pins(db, user):
|
||||
card = _shelf_card(db, user, pin)
|
||||
pins = list_pins(db, user)
|
||||
if not include_plex:
|
||||
pins = [p for p in pins if all(getattr(p, c) is None for c in _PLEX_COLUMNS)]
|
||||
batch = _Batch.load(db, user, pins)
|
||||
for pin in pins:
|
||||
card = _shelf_card(db, user, pin, batch)
|
||||
if card is not None:
|
||||
out.append(card)
|
||||
return out
|
||||
@@ -171,5 +251,17 @@ def shelf(db: Session, user: User) -> list[dict]:
|
||||
|
||||
def playables(shelf_cards: list[dict]) -> list[dict]:
|
||||
"""The `next` of every shelf entry that has one — what the hero may draw from. A pinned channel
|
||||
with nothing unwatched left, or an empty playlist, simply is not an answer right now."""
|
||||
return [c["next"] for c in shelf_cards if c.get("next")]
|
||||
with nothing unwatched left, or an empty playlist, simply is not an answer right now.
|
||||
|
||||
Each carries `pinned_from`: the pin it was resolved from. A show's next EPISODE is not itself
|
||||
pinned, so without it the hero's menu looked the episode up, missed, and offered "Pin" under a
|
||||
kicker that says "From your shelf" — and taking it inserted a second pin. With it, the hero's
|
||||
pin row acts on the shelf entry the card came from (the owner's call, 2026-09-19: "Unpin")."""
|
||||
return [
|
||||
{
|
||||
**c["next"],
|
||||
"pinned_from": {k: c[k] for k in ("source", "kind", "id", "title")},
|
||||
}
|
||||
for c in shelf_cards
|
||||
if c.get("next")
|
||||
]
|
||||
|
||||
@@ -43,8 +43,9 @@ from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, movie_card, show_card
|
||||
from app.plex.query import apply_show_state_filter, content_libs
|
||||
|
||||
# How many rows a single rail may carry. Matches `routes/plex/discover.py`'s RAIL_LIMIT on purpose —
|
||||
# the two surfaces should feel like the same app, and a rail is a rail.
|
||||
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
|
||||
# (`routes/plex/discover.py` imports it): the two surfaces should feel like the same app, and a rail
|
||||
# is a rail. One definition, because a comment was all that held two copies in step.
|
||||
RAIL_LIMIT = 20
|
||||
|
||||
# The feed parameters a Home rail runs with. Spelled out rather than borrowed from the route's
|
||||
@@ -129,18 +130,6 @@ def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) ->
|
||||
return [r for r in out if not r["position_seconds"]][:limit]
|
||||
|
||||
|
||||
def channel_next_video(db: Session, user: User, channel_id: str) -> dict | None:
|
||||
"""The newest unwatched video of a pinned channel — what "put on that channel" resolves to."""
|
||||
# `library_source="all"`: a pinned channel means the whole channel, including videos that only
|
||||
# ever arrived through a search. The feed's "organic" default is a feed-scoping choice, not a
|
||||
# statement about what belongs to a channel.
|
||||
params = dict(_FEED_BASE, channel_ids=[channel_id], scope="all", library_source="all")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys("newest", 0, rank_expr)
|
||||
row = db.execute(query.order_by(*order_by(keys)).limit(1)).first()
|
||||
return serialize(row) if row is not None else None
|
||||
|
||||
|
||||
def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||
"""One video as a feed row, with this user's watch state.
|
||||
|
||||
@@ -158,31 +147,55 @@ def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||
return serialize(row) if row is not None else None
|
||||
|
||||
|
||||
def channel_unwatched_count(db: Session, user: User, channel_id: str) -> int:
|
||||
"""How many unwatched videos a pinned channel is holding — the tile's only honest promise, since a
|
||||
channel has no runtime of its own."""
|
||||
params = dict(_FEED_BASE, channel_ids=[channel_id], scope="all", library_source="all")
|
||||
query, _ = filtered_query(db, user, **params)
|
||||
return int(db.execute(select(func.count()).select_from(query.subquery())).scalar() or 0)
|
||||
def channels_summary(
|
||||
db: Session, user: User, channel_ids: list[str]
|
||||
) -> dict[str, tuple[int, dict | None]]:
|
||||
"""For each pinned channel: (unwatched count, its newest unwatched video) — the tile's only honest
|
||||
promise, since a channel has no runtime of its own, and what "put on that channel" resolves to.
|
||||
|
||||
ONE query for the whole shelf, not two per channel: the count and the first row per channel are
|
||||
window functions over the same filtered set. `library_source="all"`: a pinned channel means the
|
||||
whole channel, including videos that only ever arrived through a search — the feed's "organic"
|
||||
default is a feed-scoping choice, not a statement about what belongs to a channel."""
|
||||
if not channel_ids:
|
||||
return {}
|
||||
params = dict(_FEED_BASE, channel_ids=list(channel_ids), scope="all", library_source="all")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
newest = order_by(sort_keys("newest", 0, rank_expr))
|
||||
per_channel = {"partition_by": Video.channel_id}
|
||||
sub = query.add_columns(
|
||||
func.row_number().over(**per_channel, order_by=newest).label("_rn"),
|
||||
func.count().over(**per_channel).label("_total"),
|
||||
).subquery()
|
||||
rows = db.execute(select(sub).where(sub.c._rn == 1)).all()
|
||||
return {r.channel_id: (int(r._total), serialize(r)) for r in rows}
|
||||
|
||||
|
||||
def playlist_summary(db: Session, user: User, playlist_id: int) -> tuple[int, dict | None]:
|
||||
"""(item count, the row to play next) for a pinned playlist — its first not-yet-watched item, or
|
||||
its first item once the whole list is watched. The same row doubles as the tile's artwork, so a
|
||||
shelf of playlists is scannable instead of a row of blank boxes."""
|
||||
def playlists_summary(
|
||||
db: Session, user: User, playlist_ids: list[int]
|
||||
) -> dict[int, tuple[int, dict | None]]:
|
||||
"""For each pinned playlist: (item count, the row to play next) — its first not-yet-watched item,
|
||||
or its first item once the whole list is watched. The same row doubles as the tile's artwork, so
|
||||
a shelf of playlists is scannable instead of a row of blank boxes. One query for the whole shelf."""
|
||||
if not playlist_ids:
|
||||
return {}
|
||||
st = aliased(VideoState)
|
||||
rows = db.execute(
|
||||
select(*feed_columns(user, st), PlaylistItem.position)
|
||||
select(*feed_columns(user, st), PlaylistItem.playlist_id.label("_playlist"))
|
||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(st, and_(st.video_id == Video.id, st.user_id == user.id))
|
||||
.where(PlaylistItem.playlist_id == playlist_id)
|
||||
.order_by(PlaylistItem.position.asc())
|
||||
.where(PlaylistItem.playlist_id.in_(playlist_ids))
|
||||
.order_by(PlaylistItem.playlist_id, PlaylistItem.position.asc())
|
||||
).all()
|
||||
if not rows:
|
||||
return 0, None
|
||||
nxt = next((r for r in rows if (r.status or "new") not in ("watched", "hidden")), rows[0])
|
||||
return len(rows), serialize(nxt)
|
||||
grouped: dict[int, list] = {}
|
||||
for r in rows:
|
||||
grouped.setdefault(r._playlist, []).append(r)
|
||||
out: dict[int, tuple[int, dict | None]] = {}
|
||||
for pid, items in grouped.items():
|
||||
nxt = next((r for r in items if (r.status or "new") not in ("watched", "hidden")), items[0])
|
||||
out[pid] = (len(items), serialize(nxt))
|
||||
return out
|
||||
|
||||
|
||||
# ── Plex ──────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -25,7 +25,7 @@ from app.downloads.jobdb import (
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
|
||||
@@ -597,7 +597,7 @@ def _download(
|
||||
}
|
||||
if job and not job.display_name and meta.title:
|
||||
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
||||
fields["display_name"] = meta.title[:255]
|
||||
fields["display_name"] = meta.title[:DISPLAY_TEXT_MAX]
|
||||
# Atomic terminal write: finalizes ONLY while the row is still running, so a child that
|
||||
# finishes just after a route cancel/pause can't resurrect it (see finalize_if_running).
|
||||
finalized = service.finalize_if_running(db, job_id, **fields)
|
||||
|
||||
@@ -20,7 +20,7 @@ from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobStatus
|
||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||
from app.models import DISPLAY_TEXT_MAX, Channel, DownloadJob, MediaAsset, Video
|
||||
|
||||
|
||||
class EditError(Exception):
|
||||
@@ -179,7 +179,7 @@ def enqueue(
|
||||
profile_id=profile_id,
|
||||
profile_snapshot=spec,
|
||||
display_name=display_name
|
||||
or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
||||
or (asset.title[:DISPLAY_TEXT_MAX] if asset.status == "ready" and asset.title else None),
|
||||
status="queued",
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
@@ -198,10 +198,13 @@ def enqueue(
|
||||
|
||||
|
||||
def _clip_title(source_title: str | None, display_name: str | None) -> str:
|
||||
# Clipped to DISPLAY_TEXT_MAX, not to a literal: this feeds `DownloadJob.display_name` through
|
||||
# `enqueue_edit`, the same column `display_text` guards. A machine-made title, so it is clipped
|
||||
# rather than run through `display_text` — blank here means "no title", not None.
|
||||
if display_name:
|
||||
return display_name[:255]
|
||||
return display_name[:DISPLAY_TEXT_MAX]
|
||||
base = source_title or "Clip"
|
||||
return f"{base} (clip)"[:255]
|
||||
return f"{base} (clip)"[:DISPLAY_TEXT_MAX]
|
||||
|
||||
|
||||
def enqueue_edit(
|
||||
@@ -255,7 +258,7 @@ def enqueue_edit(
|
||||
source_asset_id=src_asset.id,
|
||||
edit_spec=spec,
|
||||
profile_snapshot={},
|
||||
display_name=display_name or asset.title,
|
||||
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
|
||||
status="queued",
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
|
||||
@@ -25,7 +25,7 @@ from sqlalchemy.orm import Session
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, probe, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.models import DownloadJob, MediaAsset, display_text
|
||||
|
||||
|
||||
def _title_from_filename(filename: str) -> str:
|
||||
@@ -99,7 +99,7 @@ def ingest_upload(
|
||||
source_kind="upload",
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
display_name=(display_name or "").strip() or None,
|
||||
display_name=display_text(display_name),
|
||||
status="done",
|
||||
progress=100,
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Bug reports built from debug sessions (E16): the files an issue carries, and where they go —
|
||||
downloaded as a zip, or (S4) sent to a Gitea issue tracker. The session itself — what is recorded
|
||||
and tagged — is the core module `app/debugsession.py`."""
|
||||
@@ -0,0 +1,79 @@
|
||||
"""What an issue from a debug session carries, as files — one definition for the zip export and the
|
||||
Gitea upload, so the two cannot drift apart.
|
||||
|
||||
* `issue.md` — the title and the body the user wrote and reviewed in the composer (rendered there,
|
||||
in their language; the server does not rewrite it);
|
||||
* `client.log` / `server.log` — the session's evidence (`debugsession.bundle`) minus the rows the
|
||||
user took out, one JSON object per line, browser rows and server rows apart. `.log`, not
|
||||
`.jsonl`: Gitea refuses that extension (measured, plans/e16 S0);
|
||||
* the screenshots, in the order they were taken.
|
||||
|
||||
Every row is already redacted twice (at capture, and again when stored), and nothing here adds a
|
||||
field the review list did not show.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, logsink
|
||||
from app.models import DebugScreenshot, DebugSession
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IssueFile:
|
||||
name: str
|
||||
data: bytes
|
||||
mime: str
|
||||
|
||||
|
||||
def title_of(session: DebugSession) -> str:
|
||||
# Not a sentence of ours: the composer always saves a title in the user's language, and a
|
||||
# session exported before it ever saved is named by its date alone, in no language at all.
|
||||
title = ((session.draft or {}).get("title") or "").strip()
|
||||
return title or f"{session.started_at:%Y-%m-%d %H:%M} · {session.id}"
|
||||
|
||||
|
||||
def body_of(session: DebugSession) -> str:
|
||||
return ((session.draft or {}).get("body") or "").strip()
|
||||
|
||||
|
||||
def _lines(rows) -> bytes:
|
||||
return "".join(
|
||||
json.dumps(logsink.serialize(r), ensure_ascii=False, default=str) + "\n" for r in rows
|
||||
).encode()
|
||||
|
||||
|
||||
def files(db: Session, session: DebugSession) -> list[IssueFile]:
|
||||
"""The session's issue as files: the text, the two logs, the screenshots."""
|
||||
rows, _ = debugsession.bundle(db, session, drop_excluded=True)
|
||||
text = f"# {title_of(session)}\n\n{body_of(session)}\n"
|
||||
out = [
|
||||
IssueFile("issue.md", text.encode(), "text/markdown"),
|
||||
IssueFile("client.log", _lines(r for r in rows if r.source == "web"), "text/plain"),
|
||||
IssueFile("server.log", _lines(r for r in rows if r.source != "web"), "text/plain"),
|
||||
]
|
||||
shots = db.scalars(
|
||||
select(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
.order_by(DebugScreenshot.id)
|
||||
)
|
||||
for n, shot in enumerate(shots, start=1):
|
||||
ext = debugsession.IMAGE_TYPES[shot.mime]
|
||||
name = f"{debugsession.SCREENSHOT_PREFIX}{n}.{ext}"
|
||||
out.append(IssueFile(name, shot.data, shot.mime))
|
||||
return out
|
||||
|
||||
|
||||
def zipped(items: list[IssueFile]) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
|
||||
for item in items:
|
||||
# The images are compressed already: deflating them again costs time and wins nothing.
|
||||
method = zipfile.ZIP_STORED if item.mime.startswith("image/") else zipfile.ZIP_DEFLATED
|
||||
z.writestr(item.name, item.data, compress_type=method)
|
||||
return buf.getvalue()
|
||||
@@ -0,0 +1,288 @@
|
||||
"""The Gitea issue tracker a debug session's issue goes to (E16 S4) — measured against Gitea 1.27.1
|
||||
before a line of this was written (plans/e16-debug-issue.md, S0).
|
||||
|
||||
Configured by an admin (sysconfig `issues_gitea_url` / `_repo` / `_token`, the token encrypted at
|
||||
rest). The repository must be PRIVATE: an issue carries its reporter's log — page titles, filter
|
||||
values, at Teljes (TRACE) what they typed into search — and a public repository would publish it. That is
|
||||
checked on every send, not only in the admin's "test connection", because a repository can be made
|
||||
public after the fact.
|
||||
|
||||
What an issue is made of comes from `bundle.files` (the same files the zip export carries). Sending
|
||||
is three kinds of call, measured:
|
||||
* `POST /repos/{repo}/issues` — the title and the body the user wrote;
|
||||
* `POST /repos/{repo}/issues/{n}/assets?name=…` — ONE file per request (field `attachment`);
|
||||
* `PATCH /repos/{repo}/issues/{n}` — the body again, with the screenshots shown inline once their
|
||||
links exist.
|
||||
A private repository's attachment answers 404 to a browser not signed in to Gitea, which is why the
|
||||
composer's own preview shows our copies, never these links.
|
||||
|
||||
Delivery keeps its progress in the session's draft (`delivery`: the issue number, the files still to
|
||||
upload, the links of those uploaded, whether the body shows the screenshots yet), so a failure
|
||||
half-way keeps the issue that exists and a retry does only what is missing — never a second issue.
|
||||
Every step is checkpointed (the caller's `db.commit`), so what did happen is kept whatever fails
|
||||
next — and the first checkpoint comes before the first call, with the tracker's settings read ABOVE
|
||||
it, so neither a row lock nor a read snapshot nor a pooled connection is held across the calls. The
|
||||
caller claims the session for the send first (`debugsession.claim_send`) and passes the claim's
|
||||
token in, which is what keeps a second send, an autosave or a screenshot out of the way; the claim
|
||||
is renewed by every progress report, and a report whose claim is gone raises rather than writing
|
||||
over the send that took it. `debugsession.not_uploaded` / `images_pending` read that progress.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, logs, sysconfig
|
||||
from app.issues import bundle
|
||||
from app.models import DebugSession
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
TIMEOUT_SECONDS = 20.0
|
||||
|
||||
Reason = Literal[
|
||||
"not_configured", # url, repository or token missing
|
||||
"unreachable", # no answer, or a 5xx
|
||||
"bad_token", # 401/403: the token is wrong or lacks write:issue / read:repository
|
||||
"repo_not_found", # 404 on the repository
|
||||
"public_repo", # the repository is public — an issue would publish the reporter's log
|
||||
"issues_disabled", # the repository has its issue tracker switched off
|
||||
"rejected", # any other 4xx (a file type the instance does not allow, …), or an answer that is
|
||||
# not the API's (a redirect — an http:// address behind https —, a page that is not JSON)
|
||||
]
|
||||
|
||||
|
||||
class GiteaError(Exception):
|
||||
def __init__(self, reason: Reason) -> None:
|
||||
super().__init__(reason)
|
||||
self.reason: Reason = reason
|
||||
|
||||
|
||||
def _settings(db: Session) -> tuple[str, str, str]:
|
||||
return (
|
||||
sysconfig.get_str(db, "issues_gitea_url").strip().rstrip("/"),
|
||||
sysconfig.get_str(db, "issues_gitea_repo").strip().strip("/"),
|
||||
sysconfig.get_str(db, "issues_gitea_token").strip(),
|
||||
)
|
||||
|
||||
|
||||
def target(db: Session) -> dict | None:
|
||||
"""Where a report goes, for the composer to say so — the repository — or None when no tracker is
|
||||
configured (the report can then only be downloaded)."""
|
||||
url, repo, token = _settings(db)
|
||||
return {"repo": repo} if url and repo and token else None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _client(settings: tuple[str, str, str]) -> Iterator[tuple[httpx.Client, str]]:
|
||||
# Takes the settings already read, not the session: reading them HERE opened a transaction one
|
||||
# line after `deliver` had ended one, and held it across the calls below (deep-review round 4).
|
||||
url, repo, token = settings
|
||||
if not (url and repo and token):
|
||||
raise GiteaError("not_configured")
|
||||
with httpx.Client(
|
||||
base_url=f"{url}/api/v1",
|
||||
headers={"Authorization": f"token {token}", "Accept": "application/json"},
|
||||
timeout=TIMEOUT_SECONDS,
|
||||
) as client:
|
||||
yield client, repo
|
||||
|
||||
|
||||
def _call(client: httpx.Client, method: str, path: str, **kwargs) -> httpx.Response:
|
||||
try:
|
||||
r = client.request(method, path, **kwargs)
|
||||
except httpx.HTTPError as exc:
|
||||
log.warning(
|
||||
"the issue tracker did not answer",
|
||||
extra=logs.event("issues.gitea.unreachable", path=path, error=type(exc).__name__),
|
||||
)
|
||||
raise GiteaError("unreachable") from None
|
||||
if r.status_code < 300:
|
||||
return r
|
||||
# The status only: Gitea's message can quote back what was sent — the user's own title.
|
||||
log.warning(
|
||||
"the issue tracker refused a call",
|
||||
extra=logs.event("issues.gitea.refused", path=path, status=r.status_code),
|
||||
)
|
||||
if r.status_code in (401, 403):
|
||||
raise GiteaError("bad_token")
|
||||
if r.status_code == 404:
|
||||
raise GiteaError("repo_not_found")
|
||||
if r.status_code >= 500:
|
||||
raise GiteaError("unreachable")
|
||||
raise GiteaError("rejected")
|
||||
|
||||
|
||||
def _json(r: httpx.Response, *keys: str) -> dict[str, Any]:
|
||||
"""The answer as the API's JSON, holding `keys` — anything else is refused, not a crash, so a
|
||||
step after the issue was created cannot take down the request that holds its record.
|
||||
|
||||
The CREATE call is the one place this cannot keep that record: its answer IS the record. An
|
||||
answer it cannot read there leaves the session unsent, and a retry opens another issue — the
|
||||
price of not being able to tell a refused create from an unreadable one."""
|
||||
try:
|
||||
data = r.json()
|
||||
# silent-ok: not JSON is one of the answers refused (and logged) just below.
|
||||
except ValueError:
|
||||
data = None
|
||||
if not isinstance(data, dict) or any(k not in data for k in keys):
|
||||
log.warning(
|
||||
"the issue tracker answered something that is not its API",
|
||||
extra=logs.event("issues.gitea.unexpected", status=r.status_code),
|
||||
)
|
||||
raise GiteaError("rejected")
|
||||
return data
|
||||
|
||||
|
||||
def _private_repo(client: httpx.Client, repo: str) -> dict:
|
||||
data = _json(_call(client, "GET", f"/repos/{repo}"))
|
||||
if not data.get("private"):
|
||||
raise GiteaError("public_repo")
|
||||
if not data.get("has_issues"):
|
||||
raise GiteaError("issues_disabled")
|
||||
return data
|
||||
|
||||
|
||||
def check(db: Session) -> dict:
|
||||
"""The admin's "test connection": the repository answers, is private and takes issues."""
|
||||
with _client(_settings(db)) as (client, repo):
|
||||
data = _private_repo(client, repo)
|
||||
return {"repo": data.get("full_name") or repo, "url": data.get("html_url")}
|
||||
|
||||
|
||||
def preview(db: Session, text: str) -> str:
|
||||
"""The body as Gitea itself will render it (its markdown flavour, its sanitiser).
|
||||
|
||||
**No `Context`, deliberately — a repository context here does not fail the call, it kills the
|
||||
tracker.** With one, Gitea's post-processor turns every hex-shaped word into a commit link, and
|
||||
on the API's render path the commit checker it asks is nil: the whole Gitea process segfaults
|
||||
and restarts, taking the git server (and, here, the container registry) down with it. Every
|
||||
report carries hex — a tab id, a request id — so this was not an edge case but the only case.
|
||||
Measured on throwaway instances (2026-09-20): `{"Mode": "gfm", "Context": "<repo>"}` with
|
||||
`5645b1c0ae01` in the text crashes 1.26.4 and 1.27.1 and is fixed in 1.27.3 — but a self-hoster
|
||||
runs whatever version they run, so the payload stays safe here rather than relying on theirs.
|
||||
Dropping `Context` is not a downgrade of the preview: measured against the same body, the HTML
|
||||
keeps its tables, its task lists and its highlighted code blocks. It only stops resolving
|
||||
repo-relative links (a hash, a `#123`), which the issue page itself still resolves.
|
||||
"""
|
||||
with _client(_settings(db)) as (client, _repo):
|
||||
r = _call(client, "POST", "/markdown", json={"Text": text, "Mode": "gfm"})
|
||||
return r.text
|
||||
|
||||
|
||||
def deliver(
|
||||
db: Session,
|
||||
session: DebugSession,
|
||||
*,
|
||||
token: str,
|
||||
checkpoint: Callable[[], None],
|
||||
before_create: Callable[[], None] | None = None,
|
||||
on_created: Callable[[], None] | None = None,
|
||||
) -> str:
|
||||
"""Send the session's issue, or finish sending it; returns its URL.
|
||||
|
||||
`token` is the caller's claim on this session (`debugsession.claim_send`). Every progress report
|
||||
renews it and checks the row still holds it, so a send that outlived its lease stops here rather
|
||||
than writing over the one that took over (`debugsession.SendClaimLost`).
|
||||
|
||||
`checkpoint` persists what has happened so far and is called after every step (the caller's
|
||||
commit — this module does not decide when a transaction ends). `before_create` runs right before
|
||||
the tracker is asked to open a NEW issue, after the checks that can refuse it without asking,
|
||||
and may raise to stop it; `on_created` runs right after one was opened, in time to be committed
|
||||
by the SAME checkpoint that first writes `issue_url` — anything the caller must not be able to
|
||||
lose separately from the link belongs there.
|
||||
"""
|
||||
state = debugsession.delivery(session)
|
||||
files = bundle.files(db, session)
|
||||
attachments = [f for f in files if f.name != "issue.md"]
|
||||
settings = _settings(db)
|
||||
# Reading the bundle and the tracker's settings opened a transaction; end it before the first
|
||||
# call, so nothing holds a snapshot (and an idle connection) across minutes of tracker I/O.
|
||||
# The settings are read HERE and handed to `_client`, not read inside it: reading them there
|
||||
# was a SELECT one line after this commit, which reopened the transaction and held it — idle,
|
||||
# with its snapshot pinned — across the repository check and the create (deep-review round 4
|
||||
# measured it in pg_stat_activity). A comment saying otherwise sat right here.
|
||||
checkpoint()
|
||||
with _client(settings) as (client, repo):
|
||||
_private_repo(client, repo)
|
||||
if "number" not in state:
|
||||
if before_create is not None:
|
||||
before_create()
|
||||
created = _json(
|
||||
_call(
|
||||
client,
|
||||
"POST",
|
||||
f"/repos/{repo}/issues",
|
||||
json={"title": bundle.title_of(session), "body": bundle.body_of(session)},
|
||||
),
|
||||
"number",
|
||||
"html_url",
|
||||
)
|
||||
# Built ON the state read at the start so nothing already in it is dropped. The claim
|
||||
# is NOT among those things, whatever this comment used to say: `set_delivery` strips
|
||||
# any `sending` passed to it and stamps the row's own from `token` (round 4 — removing
|
||||
# the spread left the whole suite green, because the claim never travelled here).
|
||||
state = {
|
||||
**state,
|
||||
"number": created["number"],
|
||||
"pending": [f.name for f in attachments],
|
||||
"links": {},
|
||||
}
|
||||
session.issue_url = created["html_url"]
|
||||
try:
|
||||
debugsession.set_delivery(db, session, state, token)
|
||||
except debugsession.SendClaimLost:
|
||||
# The one step where losing the claim costs something irreversible: the issue is
|
||||
# already open on the tracker and this is what would have recorded its link. We
|
||||
# still must not write — another send owns the row and is opening its own issue —
|
||||
# so say which issue was left behind, because nothing else will.
|
||||
# noqa TRY400: the traceback is one SendClaimLost carrying the session id and
|
||||
# nothing else; the issue number below is the whole news.
|
||||
log.error( # noqa: TRY400
|
||||
"an issue was created by a send that had already lost its claim",
|
||||
extra=logs.event(
|
||||
"issues.gitea.orphaned",
|
||||
number=created["number"],
|
||||
session_id=session.id,
|
||||
),
|
||||
)
|
||||
raise
|
||||
if on_created is not None:
|
||||
on_created()
|
||||
checkpoint()
|
||||
log.info(
|
||||
"issue created",
|
||||
extra=logs.event("issues.gitea.created", number=created["number"]),
|
||||
)
|
||||
number = state["number"]
|
||||
for item in attachments:
|
||||
if item.name not in state["pending"]:
|
||||
continue
|
||||
asset = _json(
|
||||
_call(
|
||||
client,
|
||||
"POST",
|
||||
f"/repos/{repo}/issues/{number}/assets",
|
||||
params={"name": item.name},
|
||||
files={"attachment": (item.name, item.data, item.mime)},
|
||||
),
|
||||
"browser_download_url",
|
||||
)
|
||||
state = {
|
||||
**state,
|
||||
"pending": [n for n in state["pending"] if n != item.name],
|
||||
"links": {**state["links"], item.name: asset["browser_download_url"]},
|
||||
}
|
||||
debugsession.set_delivery(db, session, state, token)
|
||||
checkpoint()
|
||||
shots = [n for n in state["links"] if n.startswith(debugsession.SCREENSHOT_PREFIX)]
|
||||
if shots and not state.get("images_shown"):
|
||||
images = "\n".join(f"" for n in sorted(shots))
|
||||
body = f"{bundle.body_of(session)}\n\n---\n\n{images}"
|
||||
_call(client, "PATCH", f"/repos/{repo}/issues/{number}", json={"body": body})
|
||||
debugsession.set_delivery(db, session, {**state, "images_shown": True}, token)
|
||||
checkpoint()
|
||||
return session.issue_url or ""
|
||||
+48
-13
@@ -100,12 +100,14 @@ class _RequestScope:
|
||||
request; every copied context shares the same object, so what the dependency writes is seen by
|
||||
the handler and by the request line itself."""
|
||||
|
||||
__slots__ = ("user_id", "level")
|
||||
__slots__ = ("user_id", "level", "debug_session_id")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.user_id: int | None = None
|
||||
# The user's consented diagnostics level as a logging number; None at the base level.
|
||||
self.level: int | None = None
|
||||
# The user's own open debug session this request named (E16), else None.
|
||||
self.debug_session_id: str | None = None
|
||||
|
||||
|
||||
_scope: contextvars.ContextVar[_RequestScope | None] = contextvars.ContextVar(
|
||||
@@ -113,11 +115,23 @@ _scope: contextvars.ContextVar[_RequestScope | None] = contextvars.ContextVar(
|
||||
)
|
||||
|
||||
|
||||
def set_user(user_id: int | None, level: str = "INFO", until: datetime | None = None) -> None:
|
||||
def set_user(
|
||||
user_id: int | None,
|
||||
level: str = "INFO",
|
||||
until: datetime | None = None,
|
||||
debug_session: str | None = None,
|
||||
) -> None:
|
||||
"""Attribute the rest of this request's log lines to `user_id`, and log them at the user's
|
||||
consented diagnostics `level` until `until` (E22 S5; "INFO" = no elevation). Called from the
|
||||
auth dependency — the one place the session is actually resolved into an account — rather than
|
||||
from the request middleware, which runs OUTSIDE the session middleware."""
|
||||
from the request middleware, which runs OUTSIDE the session middleware.
|
||||
|
||||
`debug_session` tags every line with the user's open debug session (E16). The caller has
|
||||
already checked it is theirs and open; a session's level arrives here as `level`/`until`.
|
||||
|
||||
The root logger's floor is tracked per elevation: a request without the session's header (an
|
||||
`<img>`, a `<video>` — the browser sends those itself) carries only the user's own level, and
|
||||
must not lower the floor a concurrent request of the session still needs."""
|
||||
_user_id.set(user_id)
|
||||
scope = _scope.get()
|
||||
if scope is None: # outside log_requests (a script, a unit test): this context only
|
||||
@@ -125,9 +139,11 @@ def set_user(user_id: int | None, level: str = "INFO", until: datetime | None =
|
||||
_scope.set(scope)
|
||||
elevated = level != "INFO" and until is not None and until > datetime.now(timezone.utc)
|
||||
scope.user_id = user_id
|
||||
scope.debug_session_id = debug_session
|
||||
scope.level = resolve_level(level) if elevated else None
|
||||
if user_id is not None:
|
||||
_track_elevation(user_id, scope.level, until)
|
||||
key = user_id if debug_session is None else (user_id, debug_session)
|
||||
_track_elevation(key, scope.level, until)
|
||||
|
||||
|
||||
def set_job(job_id: int | None) -> None:
|
||||
@@ -328,6 +344,12 @@ class ContextFilter(logging.Filter):
|
||||
scope = _scope.get()
|
||||
if scope is not None and scope.user_id is not None and not hasattr(record, "user_id"):
|
||||
record.user_id = scope.user_id
|
||||
if (
|
||||
scope is not None
|
||||
and scope.debug_session_id is not None
|
||||
and not hasattr(record, "debug_session_id")
|
||||
):
|
||||
record.debug_session_id = scope.debug_session_id
|
||||
return True
|
||||
|
||||
|
||||
@@ -444,7 +466,6 @@ _THIRD_PARTY_LEVELS = {
|
||||
"httpx": "WARNING",
|
||||
"httpcore": "WARNING",
|
||||
"asyncio": "WARNING",
|
||||
"multipart": "WARNING",
|
||||
"PIL": "WARNING",
|
||||
"alembic": "INFO",
|
||||
# uvicorn's own startup/shutdown/WebSocket lines are worth keeping; its ACCESS log is not —
|
||||
@@ -545,18 +566,21 @@ def current_level() -> str:
|
||||
# The base level (`LOG_LEVEL`, then the admin knob). NOTSET until configure_logging runs — a unit
|
||||
# test, a script — so the gate lets everything through there, as it did before the gate existed.
|
||||
_base_level = logging.NOTSET
|
||||
# user id → (level number, expiry) for every elevation this process has seen and not yet expired.
|
||||
_elevations: dict[int, tuple[int, datetime]] = {}
|
||||
# → (level number, expiry) for every elevation this process has seen and not yet expired: a user's
|
||||
# own, by user id, and a debug session's (E16), by (user id, session id) — its requests alone carry
|
||||
# its level, so it is tracked apart from the user's own requests.
|
||||
_ElevationKey = int | tuple[int, str]
|
||||
_elevations: dict[_ElevationKey, tuple[int, datetime]] = {}
|
||||
_elevations_lock = threading.Lock()
|
||||
_root_applied: int | None = None
|
||||
|
||||
|
||||
def _track_elevation(user_id: int, level: int | None, until: datetime | None) -> None:
|
||||
def _track_elevation(key: _ElevationKey, level: int | None, until: datetime | None) -> None:
|
||||
with _elevations_lock:
|
||||
if level is None or until is None:
|
||||
_elevations.pop(user_id, None)
|
||||
_elevations.pop(key, None)
|
||||
else:
|
||||
_elevations[user_id] = (level, until)
|
||||
_elevations[key] = (level, until)
|
||||
active = bool(_elevations)
|
||||
if active or _root_applied != _base_level:
|
||||
_apply_root_level()
|
||||
@@ -573,8 +597,8 @@ def _apply_root_level(force: bool = False) -> None:
|
||||
# different answer would otherwise apply them in either order, and the older one could win —
|
||||
# the root back at the base while an elevation is still active.
|
||||
with _elevations_lock:
|
||||
for user_id in [u for u, (_, until) in _elevations.items() if until <= now]:
|
||||
del _elevations[user_id]
|
||||
for key in [k for k, (_, until) in _elevations.items() if until <= now]:
|
||||
del _elevations[key]
|
||||
lowest = min([_base_level, *(lvl for lvl, _ in _elevations.values())])
|
||||
if force or lowest != _root_applied:
|
||||
logging.getLogger().setLevel(lowest)
|
||||
@@ -611,13 +635,24 @@ def forget_user(user_id: int) -> None:
|
||||
del _erased[stale]
|
||||
_erased[user_id] = now
|
||||
with _elevations_lock:
|
||||
_elevations.pop(user_id, None)
|
||||
for key in [
|
||||
k for k in _elevations if k == user_id or (isinstance(k, tuple) and k[0] == user_id)
|
||||
]:
|
||||
del _elevations[key]
|
||||
# That pop can have removed the lowest elevation, and the root level is the lowest of the base
|
||||
# and every unexpired one. Recomputed here rather than left to whichever later request happens
|
||||
# to call `_track_elevation` — the invariant `_apply_root_level` documents holds at every pop.
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def end_session_elevation(session_id: str) -> None:
|
||||
"""A debug session was stopped: its elevation ends now, not at its cap."""
|
||||
with _elevations_lock:
|
||||
for key in [k for k in _elevations if isinstance(k, tuple) and k[1] == session_id]:
|
||||
del _elevations[key]
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def is_erased(user_id: int | None) -> bool:
|
||||
if user_id is None:
|
||||
return False
|
||||
|
||||
@@ -44,6 +44,7 @@ from app import logs
|
||||
from app.config import settings
|
||||
from app.db import engine
|
||||
from app.models import LogEvent
|
||||
from app.utils import iso
|
||||
|
||||
QUEUE_MAX = 10_000
|
||||
BATCH_MAX = 500
|
||||
@@ -343,3 +344,24 @@ def gc(db: Session, retention_days: int) -> int:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, retention_days))
|
||||
result = db.execute(delete(LogEvent).where(LogEvent.ts < cutoff))
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def serialize(row: LogEvent) -> dict:
|
||||
"""A stored row as the API sends it — one shape for every reader: the admin console (E22.6) and
|
||||
a debug session's evidence list (E16)."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"ts": iso(row.ts),
|
||||
"level": row.level,
|
||||
"source": row.source,
|
||||
"logger": row.logger,
|
||||
"event": row.event,
|
||||
"msg": row.msg,
|
||||
"attrs": row.attrs,
|
||||
"exc": row.exc,
|
||||
"request_id": row.request_id,
|
||||
"user_id": row.user_id,
|
||||
"job_id": row.job_id,
|
||||
"debug_session_id": row.debug_session_id,
|
||||
"version": row.version,
|
||||
}
|
||||
|
||||
+33
-10
@@ -23,6 +23,7 @@ from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.exc import DataError
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from app import bodylimit
|
||||
from app import logsink
|
||||
from app import state
|
||||
from app import sysconfig
|
||||
@@ -37,6 +38,7 @@ from app.routes import (
|
||||
auth as auth_routes,
|
||||
channels,
|
||||
config as config_routes,
|
||||
debug_sessions as debug_sessions_routes,
|
||||
diagnostics as diagnostics_routes,
|
||||
discover as discover_routes,
|
||||
downloads,
|
||||
@@ -188,15 +190,6 @@ app.add_middleware(
|
||||
https_only=settings.session_https_only, # Secure flag when served over HTTPS (prod)
|
||||
)
|
||||
|
||||
if settings.frontend_origin:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[settings.frontend_origin],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Paths reachable while the instance is unconfigured (setup mode). Everything else under /api or
|
||||
# /auth is locked until the wizard finishes; static/SPA always loads so the wizard page can render.
|
||||
_SETUP_OPEN_PREFIXES = ("/api/setup", "/api/version")
|
||||
@@ -216,6 +209,34 @@ async def setup_gate(request, call_next):
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# SEC-20 — the global request-body cap (app/bodylimit.py). FastAPI reads a declared body IN FULL
|
||||
# before the first dependency, so without this the anonymous login route is a free memory/time sink.
|
||||
#
|
||||
# WHERE it sits is the whole design, and it is pinned by `test_body_limit.py` rather than by this
|
||||
# comment, because `add_middleware` inverts the reading order (last registered = OUTERMOST) and a
|
||||
# stack described in prose drifts. Registered here it lands directly INSIDE CORS and directly
|
||||
# OUTSIDE the setup gate, which is the narrowest position that satisfies both constraints:
|
||||
#
|
||||
# * inside CORS and the security headers, so the 413 is decorated like every other response. A
|
||||
# refusal the browser drops for a missing Access-Control-Allow-Origin is no better than a hang,
|
||||
# and `capped_body`'s 413 one layer down IS decorated — two shapes for one fact is exactly what
|
||||
# bodylimit.py exists to avoid.
|
||||
# * outside the setup gate, the session and every auth dependency under the router — none of which
|
||||
# run for a body that is already too large. The setup gate matters specifically: on an instance
|
||||
# that is not yet configured it opens a DB session per request, which is precisely the anonymous
|
||||
# pre-auth work SEC-20 exists to remove.
|
||||
app.add_middleware(bodylimit.BodyLimitMiddleware)
|
||||
|
||||
if settings.frontend_origin:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[settings.frontend_origin],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def static_cache_headers(request, call_next):
|
||||
"""Long-cache the content-hashed SPA bundles. Vite hashes their filename, so a given
|
||||
@@ -258,7 +279,8 @@ async def security_headers(request, call_next):
|
||||
|
||||
# The access log (app/logs.py): request id, route template, status, duration — one line, never a
|
||||
# query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole
|
||||
# request, including what the three above do. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# request, including what every layer above does — the body cap included, so a 413 is still logged
|
||||
# with its request id and duration. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# which is why the account comes from `request.state.user_id` (set by the auth dependency, the one
|
||||
# place a session is actually resolved into an account) rather than from `request.session`.
|
||||
app.middleware("http")(log_requests)
|
||||
@@ -273,6 +295,7 @@ app.include_router(discover_routes.router)
|
||||
app.include_router(search_routes.router)
|
||||
app.include_router(me.router)
|
||||
app.include_router(diagnostics_routes.router)
|
||||
app.include_router(debug_sessions_routes.router)
|
||||
app.include_router(notifications.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(channels.router)
|
||||
|
||||
+87
-4
@@ -12,6 +12,7 @@ from sqlalchemy import (
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
LargeBinary,
|
||||
SmallInteger,
|
||||
String,
|
||||
Text,
|
||||
@@ -22,6 +23,34 @@ from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
|
||||
from app.db import Base
|
||||
|
||||
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
||||
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
||||
# `User.display_name` (their Google profile name), `DownloadJob.display_name` and
|
||||
# `DownloadJob.display_uploader`. A value wider than the column is a psycopg DataError, which
|
||||
# `main.py` re-raises for everything but the NUL case — i.e. a bare 500 for a name somebody pasted.
|
||||
DISPLAY_TEXT_MAX = 255
|
||||
|
||||
|
||||
def display_text(raw: str | None) -> str | None:
|
||||
"""A user-supplied display string on its way into one of those columns: stripped, truncated,
|
||||
blank -> None.
|
||||
|
||||
One definition because there are SIX writers and they had drifted: `POST /api/downloads`,
|
||||
`POST /api/downloads/edit`, `POST /api/downloads/upload`, the metadata PATCH's two fields, and
|
||||
the Google sign-in callback. Only the PATCH truncated. Truncating is the right answer rather
|
||||
than refusing: the name is a label on work the user has already done — a file already
|
||||
transferred, an account already authenticated — and losing its tail is not worth failing the
|
||||
request over.
|
||||
|
||||
`test_upload.py` holds both halves: that every route assigning one of these fields goes through
|
||||
this helper (parsed, not grepped), and that each of those routes truncates rather than 500s. A
|
||||
sweep is exactly the kind of thing that gets done five-sixths of the way, which is how this
|
||||
ended up being written twice.
|
||||
"""
|
||||
return (raw or "").strip()[:DISPLAY_TEXT_MAX] or None
|
||||
|
||||
|
||||
from app.utils import canonical_email
|
||||
|
||||
|
||||
@@ -113,7 +142,7 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
# not cleared eagerly — it simply stops counting).
|
||||
diagnostics_level: Mapped[str | None] = mapped_column(String(8))
|
||||
diagnostics_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
avatar_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
role: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
# The user's own YouTube channel id, resolved lazily (channels.list?mine=true) and
|
||||
@@ -565,7 +594,15 @@ class LogEvent(Base):
|
||||
the retention job (`log_retention_days`) bounds whatever another process writes late."""
|
||||
|
||||
__tablename__ = "log_events"
|
||||
__table_args__ = (Index("ix_log_events_user_ts", "user_id", "ts"),)
|
||||
__table_args__ = (
|
||||
Index("ix_log_events_user_ts", "user_id", "ts"),
|
||||
# E16: an issue's bundle is `WHERE debug_session_id = …`; nearly every row has none.
|
||||
Index(
|
||||
"ix_log_events_debug_session",
|
||||
"debug_session_id",
|
||||
postgresql_where="debug_session_id IS NOT NULL",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
ts: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
@@ -584,6 +621,52 @@ class LogEvent(Base):
|
||||
version: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class DebugSession(Base):
|
||||
"""One recording: Start → reproduce → Stop (E16, app/debugsession.py). While it is open, every
|
||||
request of its user that names it (`X-Debug-Session`) is logged at `level` and tagged with `id`,
|
||||
and the issue it ends in is built from the rows carrying that tag. `until` is the hard cap the
|
||||
server set at Start; an open session past it is over whether or not the sweep has closed it yet.
|
||||
One OPEN session per user (`uq_debug_sessions_open_per_user`, partial on `ended_at IS NULL`)."""
|
||||
|
||||
__tablename__ = "debug_sessions"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"uq_debug_sessions_open_per_user",
|
||||
"user_id",
|
||||
unique=True,
|
||||
postgresql_where="ended_at IS NULL",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
level: Mapped[str] = mapped_column(String(8))
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
until: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# app/debugsession.py `EndReason`: stopped by the user, or past `until`.
|
||||
ended_reason: Mapped[str | None] = mapped_column(String(16))
|
||||
issue_url: Mapped[str | None] = mapped_column(String(512))
|
||||
# The issue being written from it (E16 S3; app/debugsession.py `save_draft` holds its shape).
|
||||
draft: Mapped[dict | None] = mapped_column(JSONB)
|
||||
|
||||
|
||||
class DebugScreenshot(Base):
|
||||
"""A screenshot attached to a debug session (E16) — pasted, dropped, or captured from the pill.
|
||||
Stored in the row: bounded per session (app/debugsession.py), and gone with it by cascade."""
|
||||
|
||||
__tablename__ = "debug_screenshots"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
session_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("debug_sessions.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
mime: Mapped[str] = mapped_column(String(16))
|
||||
size: Mapped[int] = mapped_column(Integer)
|
||||
data: Mapped[bytes] = mapped_column(LargeBinary)
|
||||
|
||||
|
||||
class AppState(Base):
|
||||
"""Single-row global app state (admin-controlled)."""
|
||||
|
||||
@@ -898,11 +981,11 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||
)
|
||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
||||
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
||||
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
extra_links: Mapped[list | None] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(
|
||||
|
||||
@@ -43,7 +43,9 @@ def rollup(cards: list[dict]) -> dict:
|
||||
]
|
||||
resume = None
|
||||
if inprog:
|
||||
resume = inprog[-1] # continue the latest-started episode
|
||||
# The LAST in-progress episode in broadcast order — no timestamp is consulted. Home's
|
||||
# `discover.queries.next_episode` picks the same one, so both Resume buttons agree.
|
||||
resume = inprog[-1]
|
||||
else:
|
||||
resume = next((c for c in cards if c.get("status") != "watched"), None)
|
||||
return {
|
||||
|
||||
+80
-42
@@ -60,8 +60,10 @@ _WARMUP_SEG_TIMEOUT_S = 8.0
|
||||
# its index for that one. See `segment_name` for why this is a constant and not a comment.
|
||||
_VIDEO_VARIANT = "0"
|
||||
_SESSION_IDLE_S = 600 # reap a session with no access for this long
|
||||
# A session directory is named `{rating_key}_{int(start_s)}_{tag}_{aoff:+.2f}` (see start_session);
|
||||
# before multi-audio it was just `{rating_key}_{int(start_s)}`. `sweep_orphans` deletes ONLY names
|
||||
# A session directory is named `{rating_key}_u{user_id}_{int(start_s)}_{tag}_{aoff:+.2f}` (see
|
||||
# start_session); before per-viewer sessions it was `{rating_key}_{int(start_s)}_{tag}_{aoff:+.2f}`,
|
||||
# and before multi-audio just `{rating_key}_{int(start_s)}` — the older shapes stay sweepable, since a
|
||||
# restart leaves them behind like any other. `sweep_orphans` deletes ONLY names
|
||||
# matching one of those two shapes: `_HLS_ROOT` is admin-configurable (PLEX_HLS_DIR), so it may
|
||||
# legitimately point at a shared scratch path we do not own, and a blanket "delete every directory
|
||||
# under the root" would take the neighbours with it.
|
||||
@@ -72,7 +74,9 @@ _SESSION_IDLE_S = 600 # reap a session with no access for this long
|
||||
# non-numeric rating_key would merely leave its directory unswept, which is the safe direction to
|
||||
# fail. Covered by test_plex_stream.py — this regex is the only thing standing between the sweep
|
||||
# and someone else's data.
|
||||
_SESSION_DIR_RE = re.compile(r"^\d+_\d+(_[^_]+_[+-]\d+\.\d{2})?$")
|
||||
_SESSION_DIR_RE = re.compile(
|
||||
r"^\d+_\d+(_[^_]+_[+-]\d+\.\d{2})?$|^\d+_u\d+_\d+_[^_]+_[+-]\d+\.\d{2}$"
|
||||
)
|
||||
# When this process started. `sweep_orphans` only deletes directories older than this: everything
|
||||
# newer belongs to a session THIS run created, and since `start_session` reuses a directory path
|
||||
# for the same (rating_key, offset, tag), "stale name" is not the same as "stale directory".
|
||||
@@ -88,7 +92,16 @@ _PROCESS_START = time.time()
|
||||
_MTIME_GRACE_S = 2.0
|
||||
|
||||
_lock = threading.Lock()
|
||||
_sessions: dict[str, "HlsSession"] = {}
|
||||
# Keyed by VIEWER and item, (user_id, rating_key). Keyed by the item alone, a second account opening
|
||||
# the same film `_kill`ed the first one's session: the displaced viewer got a 404, reclaimed at its
|
||||
# playhead, and so evicted the other in turn — two people could not watch one film at once. (One
|
||||
# account on two devices still shares a session; that case ends in the player's bounded reclaim.)
|
||||
_sessions: dict[tuple[int, str], "HlsSession"] = {}
|
||||
# How many live sessions one account may hold at once (two devices, or a next item starting while the
|
||||
# last one winds down). A new one past it replaces that account's least recently used — whose waiting
|
||||
# session request then returns at once, since `wait_for` watches the killed ffmpeg — so one account
|
||||
# can neither pile up film-sized scratch copies nor park more than this many request workers.
|
||||
_MAX_SESSIONS_PER_USER = 2
|
||||
|
||||
|
||||
class HlsSession:
|
||||
@@ -102,16 +115,17 @@ class HlsSession:
|
||||
transcode: bool = False,
|
||||
hw: bool = False,
|
||||
tonemap: bool = False,
|
||||
user_id: int = 0,
|
||||
):
|
||||
# Unguessable per-session id, carried in the URLs the client is handed. Sessions are keyed
|
||||
# by rating_key alone, so a seek-restart or a SECOND viewer of the same item replaces the
|
||||
# registered one — and the old viewer's next request for `seg_12.ts`, resolved by rating_key,
|
||||
# was answered 200 with the NEW session's twelfth segment, produced from a different offset
|
||||
# Unguessable per-session id, carried in the URLs the client is handed. A seek-restart
|
||||
# replaces the viewer's registered session for the item — and the old player's next request
|
||||
# for `seg_12.ts`, resolved by (viewer, rating_key) alone, was answered 200 with the NEW session's twelfth segment, produced from a different offset
|
||||
# under `-copyts`. Wrong bytes, not an error, so the client's reclaim never fired on it
|
||||
# (deep-review round 5). A request naming a session that is no longer registered is a 404,
|
||||
# which the player recovers from by restarting at the playhead.
|
||||
self.id = secrets.token_urlsafe(9)
|
||||
self.key = key
|
||||
self.key = key # the item's rating_key (what the logs name)
|
||||
self.user_id = user_id # whose session it is: the registry key is (user_id, key)
|
||||
self.dir = directory
|
||||
self.proc = proc
|
||||
self.transcode = transcode # video re-encoded (non-h264 source), not stream-copied
|
||||
@@ -637,29 +651,47 @@ def _kill(s: HlsSession) -> None:
|
||||
shutil.rmtree(s.dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _enforce_cap(cap: int) -> None:
|
||||
# Caller holds _lock. Only TRANSCODE sessions count toward plex_max_transcodes — remux
|
||||
# (stream-copy) and direct-serve are cheap and deliberately uncapped (see config.py:
|
||||
# plex_max_transcodes, and the user-facing config hint). Drop the least-recently-accessed
|
||||
# transcode sessions over the cap, leaving any concurrent remux streams untouched.
|
||||
tx = [(key, s) for key, s in _sessions.items() if s.transcode]
|
||||
if len(tx) <= cap:
|
||||
return
|
||||
for key, s in sorted(tx, key=lambda kv: kv[1].last_access)[: len(tx) - cap]:
|
||||
# Somebody's playback stops here. The route turns the missing session into a retryable
|
||||
# 404 and the player restarts at the playhead, so from the server nothing failed — and
|
||||
# "my film keeps rebuffering" has no other explanation anywhere.
|
||||
log.info(
|
||||
"a transcode session was dropped to stay under the cap",
|
||||
extra=logs.event("plex.stream.session_culled", key=key, cap=cap, sessions=len(tx)),
|
||||
)
|
||||
_kill(s)
|
||||
_sessions.pop(key, None)
|
||||
def _enforce_caps(user_id: int, max_transcodes: int, max_sessions: int) -> None:
|
||||
"""Caller holds _lock. Three bounds, applied after a session is registered, each dropping the
|
||||
least recently accessed sessions over it:
|
||||
|
||||
* this account's own sessions (`_MAX_SESSIONS_PER_USER`) — first, so a single account filling the
|
||||
box evicts itself before anyone else;
|
||||
* TRANSCODE sessions (plex_max_transcodes): the CPU/GPU-heavy ones;
|
||||
* EVERY session (plex_max_sessions). A remux is cheap on the CPU but not on the disk: each one is a
|
||||
whole segmented copy of the film under `_HLS_ROOT`, written faster than realtime. Counted only
|
||||
by the transcode cap, N distinct items were N ffmpegs and N film-sized scratch directories.
|
||||
|
||||
Somebody's playback stops at each drop. The route turns the missing session into a retryable 404
|
||||
and the player restarts at the playhead, so from the server nothing failed — and "my film keeps
|
||||
rebuffering" has no other explanation anywhere, hence the line."""
|
||||
|
||||
def cull(pool: list[tuple[tuple[int, str], HlsSession]], cap: int, bound: str) -> None:
|
||||
if len(pool) <= cap:
|
||||
return
|
||||
for reg, sess in sorted(pool, key=lambda kv: kv[1].last_access)[: len(pool) - cap]:
|
||||
log.info(
|
||||
"a session was dropped to stay under a cap",
|
||||
extra=logs.event(
|
||||
"plex.stream.session_culled",
|
||||
key=sess.key,
|
||||
bound=bound,
|
||||
cap=cap,
|
||||
sessions=len(pool),
|
||||
),
|
||||
)
|
||||
_kill(sess)
|
||||
_sessions.pop(reg, None)
|
||||
|
||||
cull([kv for kv in _sessions.items() if kv[0][0] == user_id], _MAX_SESSIONS_PER_USER, "user")
|
||||
cull([kv for kv in _sessions.items() if kv[1].transcode], max_transcodes, "transcode")
|
||||
cull(list(_sessions.items()), max_sessions, "total")
|
||||
|
||||
|
||||
def start_session(
|
||||
db: DbSession,
|
||||
item: PlexItem,
|
||||
user_id: int,
|
||||
start_s: float,
|
||||
audio_ord: int | None = None,
|
||||
aoff: float = 0.0,
|
||||
@@ -679,6 +711,7 @@ def start_session(
|
||||
# session respawn (the HW->SW fallback) must not pick up a mid-playback config change.
|
||||
loudnorm = sysconfig.get_bool(db, "plex_audio_loudnorm")
|
||||
key = item.rating_key
|
||||
reg = (user_id, key) # the registry key: one session per viewer per item
|
||||
start_s = max(0.0, float(start_s))
|
||||
audio_langs = _probe_audio_langs(src) if multi else []
|
||||
is_multi = len(audio_langs) > 1 # only worth a master playlist when there really are ≥2 tracks
|
||||
@@ -691,7 +724,7 @@ def start_session(
|
||||
# the tag with a DASH — `_SESSION_DIR_RE` treats the tag as one underscore-free field, and that
|
||||
# regex is the only thing keeping the orphan sweep off a shared scratch path (see its comment).
|
||||
tag = ("multi" if is_multi else str(audio_ord)) + ("-ln" if loudnorm else "")
|
||||
directory = _HLS_ROOT / f"{key}_{int(start_s)}_{tag}_{aoff:+.2f}"
|
||||
directory = _HLS_ROOT / f"{key}_u{user_id}_{int(start_s)}_{tag}_{aoff:+.2f}"
|
||||
|
||||
def _seg(i: int) -> Path:
|
||||
return directory / segment_name(i, _VIDEO_VARIANT if variants else None)
|
||||
@@ -717,9 +750,9 @@ def start_session(
|
||||
# platform without fork (the API always runs in a Linux container in prod) it's simply omitted.
|
||||
preexec = _sw_child_preexec if (transcode and not hw and os.name == "posix") else None
|
||||
with _lock:
|
||||
if replace_only is not None and _sessions.get(key) is not replace_only:
|
||||
if replace_only is not None and _sessions.get(reg) is not replace_only:
|
||||
return None # superseded/culled while we waited out of lock — don't clobber the newer one
|
||||
old = _sessions.pop(key, None)
|
||||
old = _sessions.pop(reg, None)
|
||||
if old is not None:
|
||||
_kill(old)
|
||||
shutil.rmtree(directory, ignore_errors=True)
|
||||
@@ -753,8 +786,9 @@ def start_session(
|
||||
transcode=transcode,
|
||||
hw=hw,
|
||||
tonemap=tonemap,
|
||||
user_id=user_id,
|
||||
)
|
||||
_sessions[key] = s
|
||||
_sessions[reg] = s
|
||||
# The one line that says which pipeline a session actually got. Everything downstream
|
||||
# — the cap, the HW→SW fallback, the subtitle offset — is about this choice, and until
|
||||
# now it was only inferable from the ffmpeg command nobody keeps.
|
||||
@@ -770,9 +804,13 @@ def start_session(
|
||||
start_s=round(start_s, 1),
|
||||
),
|
||||
)
|
||||
# Admin-configurable concurrency cap (Configuration → Plex → Max concurrent transcodes);
|
||||
# at least 1 so playback can't be capped to zero.
|
||||
_enforce_cap(max(1, sysconfig.get_int(db, "plex_max_transcodes")))
|
||||
# Admin-configurable caps (Configuration → Plex); at least 1 each so playback can't be
|
||||
# capped to zero.
|
||||
_enforce_caps(
|
||||
user_id,
|
||||
max(1, sysconfig.get_int(db, "plex_max_transcodes")),
|
||||
max(1, sysconfig.get_int(db, "plex_max_sessions")),
|
||||
)
|
||||
return s
|
||||
|
||||
s = _spawn(*_select_encode(db, item, transcode, src))
|
||||
@@ -798,7 +836,7 @@ def start_session(
|
||||
# turns a None return into a "Media file not found on disk" 404 — dishonest for a file that
|
||||
# exists. A (dead) session instead degrades to a retryable 404 ("No active playback session" /
|
||||
# "Not available yet") the client re-requests.
|
||||
return current_session(key) or s
|
||||
return current_session(user_id, key) or s
|
||||
s = respawned
|
||||
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
||||
if ready:
|
||||
@@ -829,9 +867,9 @@ def start_session(
|
||||
return s
|
||||
|
||||
|
||||
def current_session(key: str) -> HlsSession | None:
|
||||
def current_session(user_id: int, key: str) -> HlsSession | None:
|
||||
with _lock:
|
||||
s = _sessions.get(key)
|
||||
s = _sessions.get((user_id, key))
|
||||
if s is not None:
|
||||
s.last_access = time.time()
|
||||
return s
|
||||
@@ -1060,21 +1098,21 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int:
|
||||
now = time.time()
|
||||
dropped = 0
|
||||
with _lock:
|
||||
for key, s in list(_sessions.items()):
|
||||
for reg, s in list(_sessions.items()):
|
||||
done = s.proc.poll() is not None
|
||||
idle = now - s.last_access
|
||||
if idle > _SESSION_IDLE_S or (done and idle > finished_idle_s):
|
||||
_kill(s)
|
||||
_sessions.pop(key, None)
|
||||
_sessions.pop(reg, None)
|
||||
dropped += 1
|
||||
return dropped
|
||||
|
||||
|
||||
def touch_session(key: str) -> None:
|
||||
"""Say a viewer is still there for `key`, without fetching anything. `last_access` used to move
|
||||
def touch_session(user_id: int, key: str) -> None:
|
||||
"""Say this viewer is still there for `key`, without fetching anything. `last_access` used to move
|
||||
only on an HLS file fetch, which a paused player does not do — so the only way to keep a paused
|
||||
viewer's session alive was to keep EVERY abandoned one just as long."""
|
||||
with _lock:
|
||||
s = _sessions.get(key)
|
||||
s = _sessions.get((user_id, key))
|
||||
if s is not None:
|
||||
s.last_access = time.time()
|
||||
|
||||
+151
-12
@@ -32,6 +32,7 @@ window still took the full 145 s and emitted the whole track), so windows are ex
|
||||
zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
||||
"""
|
||||
|
||||
import codecs
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
@@ -217,7 +218,11 @@ _FULL_END_S = 1_000_000_000.0 # stand-in for "to the end of the file" (JSON can
|
||||
|
||||
|
||||
def cache_key(rating_key: str, sub_ord: int) -> str:
|
||||
return f"sub_{rating_key}_{sub_ord}"
|
||||
# Versioned whenever `decode_subtitle` changes what a track decodes to — `sub2_` when legacy code
|
||||
# pages were cached as U+FFFD, `sub3_` when a stray byte, an untagged file or Serbian still went
|
||||
# to the wrong code page. The cache has no eviction, so every track is read again once rather
|
||||
# than a broken one being served for good; the older files are just left behind.
|
||||
return f"sub3_{rating_key}_{sub_ord}"
|
||||
|
||||
|
||||
def _vtt_path(key: str) -> Path:
|
||||
@@ -461,11 +466,18 @@ _FULL_POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="plex-subs-ful
|
||||
# which time something has finished.
|
||||
_MAX_WINDOWS_INFLIGHT = 8
|
||||
_MAX_FULL_INFLIGHT = 6
|
||||
# …of which one account may hold this many. The full-pass slots are minutes each, and a hostile (or
|
||||
# merely curious) account opening subtitles on film after film could otherwise hold all of them while
|
||||
# everyone else's tracks never completed. A viewer over their share still gets windows — the part
|
||||
# that makes a track watchable now — just not another whole-file read until one of theirs ends.
|
||||
_MAX_FULL_PER_USER = 3
|
||||
_MAX_TRACK_WINDOWS = 2
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
# Queued or running jobs → the absolute span a window job reads (None for a full pass).
|
||||
_INFLIGHT: dict[tuple, tuple[float, float] | None] = {}
|
||||
# rating_key → the account whose poll queued that item's full pass (see _MAX_FULL_PER_USER).
|
||||
_FULL_OWNERS: dict[str, int] = {}
|
||||
# Track-level failures: a track that cannot be served at all (no item, a bitmap track, Plex down…).
|
||||
_FAILURES: dict[str, tuple[str, float]] = {}
|
||||
# rating_key → when its full pass last failed (see FULL_RETRY_S).
|
||||
@@ -515,6 +527,7 @@ def reset_state() -> None:
|
||||
global _STOPPING
|
||||
with _LOCK:
|
||||
_INFLIGHT.clear()
|
||||
_FULL_OWNERS.clear()
|
||||
_FAILURES.clear()
|
||||
_FULL_FAILED.clear()
|
||||
_WIN_FAILED.clear()
|
||||
@@ -598,7 +611,7 @@ def _window_failed(key: str, at: float) -> bool:
|
||||
return any(a <= at <= b for a, b, _ in _live_window_failures(key))
|
||||
|
||||
|
||||
def ensure(rating_key: str, sub_ord: int, at: float = 0.0) -> dict:
|
||||
def ensure(rating_key: str, sub_ord: int, at: float = 0.0, viewer: int = 0) -> dict:
|
||||
"""State of one subtitle track, scheduling whatever extraction is still missing. NEVER blocks.
|
||||
|
||||
``ready`` = fully extracted · ``partial`` = usable, covers ``at``, still growing · ``preparing``
|
||||
@@ -641,8 +654,23 @@ def ensure(rating_key: str, sub_ord: int, at: float = 0.0) -> dict:
|
||||
if edge - at <= LOOKAHEAD_S and edge < _FULL_END_S:
|
||||
probe = edge + 1.0
|
||||
want_window = not _window_failed(key, probe)
|
||||
_schedule(rating_key, sub_ord, probe, want_window=want_window, want_full=not full_failed)
|
||||
return {"state": "partial" if covered else "preparing", "code": None, "gen": gen}
|
||||
_schedule(
|
||||
rating_key,
|
||||
sub_ord,
|
||||
probe,
|
||||
want_window=want_window,
|
||||
want_full=not full_failed,
|
||||
viewer=viewer,
|
||||
)
|
||||
# `growing`: whether the track will fill in on its own. False once the full pass has failed and
|
||||
# is backed off — a partial track then stays partial until a window is asked for, so the player
|
||||
# may poll slowly (only to read ahead of an edge) instead of every two seconds for the whole film.
|
||||
return {
|
||||
"state": "partial" if covered else "preparing",
|
||||
"code": None,
|
||||
"gen": gen,
|
||||
"growing": not full_failed,
|
||||
}
|
||||
|
||||
|
||||
def cached_track(rating_key: str, sub_ord: int) -> tuple[str | None, str | None]:
|
||||
@@ -659,7 +687,12 @@ def cached_track(rating_key: str, sub_ord: int) -> tuple[str | None, str | None]
|
||||
|
||||
|
||||
def _schedule(
|
||||
rating_key: str, sub_ord: int, at: float, want_window: bool, want_full: bool = True
|
||||
rating_key: str,
|
||||
sub_ord: int,
|
||||
at: float,
|
||||
want_window: bool,
|
||||
want_full: bool = True,
|
||||
viewer: int = 0,
|
||||
) -> None:
|
||||
"""Kick off the jobs this track still needs: the window that makes it watchable now, and the one
|
||||
full pass that makes it complete. A 1 s status poll (or a second viewer) joins a job already
|
||||
@@ -692,8 +725,15 @@ def _schedule(
|
||||
ident = ("win", rating_key, sub_ord, w_start)
|
||||
_INFLIGHT[ident] = (w_start, w_start + WINDOW_PRE_S + WINDOW_LEN_S)
|
||||
jobs.append((ident, w_start))
|
||||
if want_full and n_full < _MAX_FULL_INFLIGHT and ("full", rating_key) not in _INFLIGHT:
|
||||
mine = sum(1 for owner in _FULL_OWNERS.values() if owner == viewer)
|
||||
if (
|
||||
want_full
|
||||
and n_full < _MAX_FULL_INFLIGHT
|
||||
and ("full", rating_key) not in _INFLIGHT
|
||||
and mine < _MAX_FULL_PER_USER
|
||||
):
|
||||
_INFLIGHT[("full", rating_key)] = None
|
||||
_FULL_OWNERS[rating_key] = viewer
|
||||
jobs.append((("full", rating_key), None))
|
||||
for ident, window_start in jobs:
|
||||
pool = _FULL_POOL if window_start is None else _WIN_POOL
|
||||
@@ -757,6 +797,8 @@ def _run(
|
||||
finally:
|
||||
with _LOCK:
|
||||
_INFLIGHT.pop(ident, None)
|
||||
if ident[0] == "full":
|
||||
_FULL_OWNERS.pop(rating_key, None)
|
||||
|
||||
|
||||
def _pass_failed(
|
||||
@@ -822,10 +864,13 @@ def _extract_and_publish(rating_key: str, sub_ord: int, window_start: float | No
|
||||
for i, s in enumerate(subs)
|
||||
if s.get("index") is not None and not _is_bitmap(s)
|
||||
}
|
||||
start = window_start or 0.0
|
||||
# A full pass starts where every track's leading coverage ends: a pass that ran out of time
|
||||
# (`EXTRACT_TIMEOUT_S`) kept what it reached, and restarting from 0 each `FULL_RETRY_S` meant a file
|
||||
# too big to read in one go never completed. Windows handle an offset start already; so does this.
|
||||
start = window_start if window_start is not None else _full_pass_start(rating_key, targets)
|
||||
length = (WINDOW_LEN_S + WINDOW_PRE_S) if window_start is not None else None
|
||||
end = _FULL_END_S if length is None else start + length
|
||||
texts, clean = _ffmpeg_extract(str(src), sorted(targets), window_start, length)
|
||||
texts, clean = _ffmpeg_extract(str(src), sorted(targets), start or None, length)
|
||||
for emb_ord, vtt in texts.items():
|
||||
key = cache_key(rating_key, targets[emb_ord])
|
||||
# A pass that exited non-zero holds only what ffmpeg reached before it died. Claim no more
|
||||
@@ -846,15 +891,108 @@ def _extract_and_publish(rating_key: str, sub_ord: int, window_start: float | No
|
||||
_pass_failed(rating_key, [key], window_start, (covered_end, end))
|
||||
|
||||
|
||||
def _full_pass_start(rating_key: str, targets: dict[int, int]) -> float:
|
||||
"""Where the full pass may begin: the end of the span from 0 that EVERY target track already has
|
||||
(one read feeds them all, so the least-covered track decides), backed off by `WINDOW_PRE_S` so a
|
||||
cue running across the join is read whole — `merge_vtt` folds the overlap. 0 when any track has
|
||||
nothing from the start."""
|
||||
ends = []
|
||||
for ord_ in targets.values():
|
||||
key = cache_key(rating_key, ord_)
|
||||
ranges, _ = read_meta(key) if _exists(_vtt_path(key)) else ([], False)
|
||||
lead = [b for a, b in ranges if a <= 0.0]
|
||||
if not lead:
|
||||
return 0.0
|
||||
ends.append(max(lead))
|
||||
if not ends:
|
||||
return 0.0
|
||||
return max(0.0, float(f"{min(ends) - WINDOW_PRE_S:.3f}"))
|
||||
|
||||
|
||||
# The legacy code page a subtitle in this language (Plex's ISO 639-2 `languageCode`) was most likely
|
||||
# saved in. Serbian sits here, not with Cyrillic: its releases are overwhelmingly Latin script in
|
||||
# Windows-1250, like Croatian and Bosnian.
|
||||
_CP1250 = {
|
||||
"hun",
|
||||
"ces",
|
||||
"cze",
|
||||
"pol",
|
||||
"slk",
|
||||
"slo",
|
||||
"slv",
|
||||
"hrv",
|
||||
"bos",
|
||||
"srp",
|
||||
"ron",
|
||||
"rum",
|
||||
"sqi",
|
||||
"alb",
|
||||
}
|
||||
_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac"}
|
||||
# Western European languages keep Windows-1252 (cp1250 has no à/è/ñ). An UNTAGGED file falls to
|
||||
# Windows-1250: its ASCII and á/é/í/ó/ö/ú/ü match Windows-1252 byte for byte, and it also gets ő/ű
|
||||
# right, which Windows-1252 turns into õ/û — the library this reads is mostly Hungarian.
|
||||
_CP1252 = {
|
||||
"eng",
|
||||
"fre",
|
||||
"fra",
|
||||
"ger",
|
||||
"deu",
|
||||
"spa",
|
||||
"ita",
|
||||
"por",
|
||||
"dut",
|
||||
"nld",
|
||||
"swe",
|
||||
"dan",
|
||||
"nor",
|
||||
"fin",
|
||||
"ice",
|
||||
"isl",
|
||||
"cat",
|
||||
}
|
||||
# A file that is almost all valid UTF-8 is UTF-8 with a few stray bytes (a pasted Windows-1252 quote),
|
||||
# not a legacy file: accents in a legacy code page are nearly never valid UTF-8 sequences, so a real
|
||||
# legacy file has hardly any. At least this many valid multi-byte characters per invalid byte.
|
||||
_UTF8_VALID_PER_STRAY = 10
|
||||
|
||||
|
||||
def decode_subtitle(raw: bytes, language_code: str | None) -> str:
|
||||
"""A sidecar subtitle file as text. SubRip carries no charset, and a Hungarian release is
|
||||
typically Windows-1250: decoded as UTF-8 with `replace`, every ő/ű/é came out as U+FFFD (found
|
||||
live, 1391 of them in one episode). So: a BOM decides; else valid UTF-8 stays UTF-8 (a legacy
|
||||
code page with accents is almost never valid UTF-8), and so does NEARLY valid UTF-8 — one stray
|
||||
byte must not send a whole UTF-8 file to a legacy code page; else the language's legacy code page."""
|
||||
if raw.startswith(codecs.BOM_UTF8):
|
||||
return raw[len(codecs.BOM_UTF8) :].decode("utf-8", "replace")
|
||||
if raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
|
||||
return raw.decode("utf-16", "replace")
|
||||
try:
|
||||
return raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
# silent-ok: not strictly UTF-8 — nearly-UTF-8 or a legacy code page, told apart below.
|
||||
pass
|
||||
lenient = raw.decode("utf-8", "replace")
|
||||
stray = lenient.count("�")
|
||||
valid = sum(1 for ch in lenient if ch > "" and ch != "�")
|
||||
if valid >= stray * _UTF8_VALID_PER_STRAY:
|
||||
return lenient
|
||||
lang = (language_code or "").lower()
|
||||
legacy = "cp1251" if lang in _CP1251 else "cp1252" if lang in _CP1252 else "cp1250"
|
||||
return raw.decode(legacy, "replace")
|
||||
|
||||
|
||||
def _sidecar_vtt(db: Session, stream: dict) -> str:
|
||||
"""An external subtitle file, fetched through Plex and converted to WebVTT."""
|
||||
"""An external subtitle file, fetched through Plex and converted to WebVTT. Its bytes are made
|
||||
UTF-8 first (`decode_subtitle`), on the ffmpeg path too: ffmpeg reads a text subtitle as UTF-8."""
|
||||
with PlexClient(db) as plex:
|
||||
raw = plex.raw_get(stream["key"])
|
||||
text = decode_subtitle(raw, stream.get("languageCode"))
|
||||
codec = (stream.get("codec") or stream.get("format") or "").lower()
|
||||
if codec in ("srt", "subrip", ""):
|
||||
return srt_to_vtt(raw.decode("utf-8", "replace"))
|
||||
return srt_to_vtt(text)
|
||||
with tempfile.NamedTemporaryFile(suffix="." + (codec or "sub"), delete=False) as fh:
|
||||
fh.write(raw)
|
||||
fh.write(text.encode("utf-8"))
|
||||
tmp = fh.name
|
||||
try:
|
||||
return ffmpeg_to_vtt(tmp, None)
|
||||
@@ -884,9 +1022,10 @@ def _ffmpeg_extract(
|
||||
cmd += ["-map", f"0:s:{emb_ord}"]
|
||||
if length is not None:
|
||||
cmd += ["-t", f"{length:.3f}"]
|
||||
if start:
|
||||
# The input seek lands on the keyframe at or BEFORE `start`, so a cue already running
|
||||
# at `start` comes out at a negative time — and by default the muxer then shifts the
|
||||
# WHOLE output so that cue sits at 0: every cue of the window late by the distance to
|
||||
# WHOLE output so that cue sits at 0: every cue of the read late by the distance to
|
||||
# that keyframe (1–5 s measured on a real library file, a whole GOP on a synthetic
|
||||
# one), and doubled for good once the full pass merged in beside it. Kept negative, the
|
||||
# times are exact on the zero-based clock (`_vtt_ts_to_s` reads the sign).
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""A request the app refuses, said in a way the UI can put in the reader's language.
|
||||
|
||||
Every refusal the API answers carries a structured `detail` — `{code, reason, …params}` — and the client
|
||||
translates it (`lib/api/core.ts` `localizeDetail`). A bare English string reached the Hungarian UI
|
||||
verbatim: the blocking error dialog, and every inline surface that printed `HttpError.detail`, showed
|
||||
whatever sentence the route happened to hand-write. `refused` is the generic family
|
||||
(`errors.refused.<reason>`, the params interpolated into the sentence); the older specific codes
|
||||
(`not_found`, `youtube`, `playlist`, the admin refusals, …) stay, since each is already translated.
|
||||
|
||||
A test (`test_http_details.py`) reads every `HTTPException` in the app and fails on a string detail,
|
||||
and the frontend's contract test requires every `refused(…, "<reason>")` to be named in both
|
||||
languages — so the rule is enforced, not described.
|
||||
"""
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
def refused(status: int, reason: str, **params: object) -> HTTPException:
|
||||
"""The refusal to raise: `raise refused(409, "view_name_taken", name=name)`. `params` travel in
|
||||
the detail for the sentence to interpolate (`{{name}}`); keep them to values the reader may see."""
|
||||
return HTTPException(status_code=status, detail={"code": "refused", "reason": reason, **params})
|
||||
|
||||
|
||||
# `sysconfig.InvalidSetting.reason` → the refusal it answers as. A dict of literals, so the frontend's
|
||||
# contract test can read every reason this can produce.
|
||||
_SETTING_REASONS = {
|
||||
"not_a_number": "setting_not_a_number",
|
||||
"too_low": "setting_too_low",
|
||||
"too_high": "setting_too_high",
|
||||
"not_a_choice": "setting_not_a_choice",
|
||||
"no_encryption_key": "no_encryption_key",
|
||||
}
|
||||
|
||||
|
||||
def setting_refused(exc) -> HTTPException:
|
||||
"""A value the admin entered that `sysconfig.set_value` refused (`exc`: its `InvalidSetting`)."""
|
||||
return refused(400, _SETTING_REASONS[exc.reason], key=exc.key, **exc.params)
|
||||
@@ -6,13 +6,16 @@ and "fetch a row this user owns or 404" idioms have one definition, one status c
|
||||
one existence-leak policy.
|
||||
"""
|
||||
|
||||
from typing import Any, TypeVar
|
||||
from typing import IO, Any, TypeVar
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi import HTTPException, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import Base
|
||||
from app.models import User
|
||||
from app.issues import gitea
|
||||
from app.refusal import refused
|
||||
from app.youtube.client import YouTubeError, refusal_reason
|
||||
|
||||
M = TypeVar("M", bound=Base)
|
||||
|
||||
@@ -25,10 +28,13 @@ def not_found(what: str = "generic") -> HTTPException:
|
||||
return HTTPException(status_code=404, detail={"code": "not_found", "reason": what})
|
||||
|
||||
|
||||
def get_or_404(db: Session, model: type[M], id_: Any, *, what: str = "generic") -> M:
|
||||
def get_or_404(
|
||||
db: Session, model: type[M], id_: Any, *, what: str = "generic", lock: bool = False
|
||||
) -> M:
|
||||
"""Fetch a row by primary key or raise 404. Replaces the `db.get(Model, id); if None: 404`
|
||||
idiom repeated across routes."""
|
||||
obj = db.get(model, id_)
|
||||
idiom repeated across routes. `lock` takes it FOR UPDATE (bypassing the identity map), so two
|
||||
requests that read-then-write its children (a playlist's item positions) run one after the other."""
|
||||
obj = db.get(model, id_, with_for_update=True if lock else None)
|
||||
if obj is None:
|
||||
raise not_found(what)
|
||||
return obj
|
||||
@@ -42,12 +48,98 @@ def owned_or_404(
|
||||
*,
|
||||
owner_attr: str = "user_id",
|
||||
what: str = "generic",
|
||||
lock: bool = False,
|
||||
) -> M:
|
||||
"""Fetch a row by primary key that `user` owns, or raise 404. A row owned by someone
|
||||
"""Fetch a row by primary key that `user` owns, or raise 404 (`lock`: see `get_or_404`). A row owned by someone
|
||||
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
||||
Generalizes the near-identical per-route ownership helpers (e.g. plex's
|
||||
`_own_playlist_or_404`, the saved-views owner check)."""
|
||||
obj = get_or_404(db, model, id_, what=what)
|
||||
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
|
||||
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
|
||||
obj = get_or_404(db, model, id_, what=what, lock=lock)
|
||||
if getattr(obj, owner_attr) != user.id:
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
|
||||
def youtube_refusal(exc: Exception) -> HTTPException:
|
||||
"""The answer to an action YouTube refused: a 422 whose reason the UI translates
|
||||
(`errors.youtube.<reason>`). One definition, because the routes had grown three shapes of it —
|
||||
a bare English string quoting the upstream message, a playlist-scoped reason, and nothing at
|
||||
all (a raw 500 from the manual syncs). The upstream failure is already logged where it
|
||||
happened (`youtube.api.error`). A failure that is not the Data API's (the search scraper's) is
|
||||
YouTube being unavailable to us — the retryable answer."""
|
||||
reason = refusal_reason(exc) if isinstance(exc, YouTubeError) else "unavailable"
|
||||
return HTTPException(status_code=422, detail={"code": "youtube", "reason": reason})
|
||||
|
||||
|
||||
async def capped_body(request: Request, limit: int) -> bytes:
|
||||
"""The request body, refused (413 `too_large`) the moment it passes `limit` bytes while still
|
||||
arriving. Counted, not taken from Content-Length: a declared length can be absent (chunked) or
|
||||
wrong, and counting stops reading at the same point a declared-length check would have."""
|
||||
body = bytearray()
|
||||
async for chunk in request.stream():
|
||||
body += chunk
|
||||
if len(body) > limit:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
return bytes(body)
|
||||
|
||||
|
||||
async def capped_stream_to(
|
||||
request: Request, out: IO[bytes], limit: int, refusal: HTTPException
|
||||
) -> int:
|
||||
"""`capped_body`'s sibling for a body too large to hold: the same counting rule, written
|
||||
straight to `out` instead of accumulated, returning the size.
|
||||
|
||||
Two deliberate differences from `capped_body`. It never builds a `bytearray` — the upload route
|
||||
admits 500 MiB by default, so holding the body would trade a disk problem for a memory one. And
|
||||
the refusal is the CALLER'S, passed in rather than assumed: `capped_body`'s flat 413 is right
|
||||
for a screenshot, while the upload route answers 422 `upload_too_large` carrying the admin
|
||||
knob's own number, which is what its dialog shows the user. One counting rule, two answers,
|
||||
neither of them a second copy of the loop.
|
||||
|
||||
The caller owns `out` and owns cleaning it up: a body refused half-way has already written what
|
||||
arrived before the limit was passed.
|
||||
|
||||
**The refusal DRAINS the rest of the body before it is raised, and that is not politeness.**
|
||||
Answering mid-upload and closing the connection is what the client sees as a network failure,
|
||||
not as a refusal: the browser's XHR fires `onerror` before it can read the response, so a file
|
||||
over the limit reported itself as "the server couldn't be reached" instead of naming the limit —
|
||||
measured in a real browser, not reasoned about. Reading the remainder to nowhere lets the client
|
||||
finish sending, read the 422 and say the true thing.
|
||||
|
||||
The bytes are discarded, never written, and the whole request is already bounded by
|
||||
`bodylimit`'s per-route ceiling — so the worst case is bandwidth an authenticated caller was
|
||||
sending anyway, against the old behaviour which wrote exactly those bytes to disk."""
|
||||
size = 0
|
||||
over = False
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if over:
|
||||
continue # draining — see the docstring; counted, not stored
|
||||
if size > limit:
|
||||
over = True
|
||||
continue
|
||||
out.write(chunk)
|
||||
if over:
|
||||
raise refusal
|
||||
return size
|
||||
|
||||
|
||||
def issue_refused(exc: gitea.GiteaError) -> HTTPException:
|
||||
"""The refusal an issue-tracker failure (E16) answers as — the send, the preview and the
|
||||
admin's connection test alike. Literal reasons, one per case: the frontend's contract test reads
|
||||
them to require a sentence for each in both languages."""
|
||||
match exc.reason:
|
||||
case "not_configured":
|
||||
return refused(400, "issues_not_configured")
|
||||
case "unreachable":
|
||||
return refused(422, "issues_unreachable")
|
||||
case "bad_token":
|
||||
return refused(422, "issues_bad_token")
|
||||
case "repo_not_found":
|
||||
return refused(422, "issues_repo_not_found")
|
||||
case "public_repo":
|
||||
return refused(422, "issues_public_repo")
|
||||
case "issues_disabled":
|
||||
return refused(422, "issues_disabled")
|
||||
case _:
|
||||
return refused(422, "issues_rejected")
|
||||
|
||||
+27
-25
@@ -39,6 +39,7 @@ from app.routes.auth import (
|
||||
send_verification_link,
|
||||
)
|
||||
from app.utils import canonical_email, iso, valid_email
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
|
||||
@@ -169,7 +170,7 @@ def _activate_user_by_email(db: Session, email: str) -> None:
|
||||
user = next((u for u in candidates if u.email == email), None) or next(iter(candidates), None)
|
||||
if user is not None and not user.is_active:
|
||||
user.is_active = True
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
|
||||
|
||||
def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
@@ -179,7 +180,7 @@ def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
# sending a duplicate decision mail, even flipping an already-approved invite to "denied" and
|
||||
# mailing a rejection to an already-active account.
|
||||
if inv.status != "pending":
|
||||
raise HTTPException(status_code=409, detail="Invite already decided")
|
||||
raise refused(409, "invite_decided")
|
||||
inv.status = "approved" if approved else "denied"
|
||||
inv.decided_at = datetime.now(timezone.utc)
|
||||
inv.decided_by = admin.email
|
||||
@@ -191,7 +192,7 @@ def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
target_id=inv.id,
|
||||
summary=inv.email,
|
||||
)
|
||||
db.commit()
|
||||
db.flush() # the handler commits, with the activation that goes with it
|
||||
return inv
|
||||
|
||||
|
||||
@@ -210,6 +211,9 @@ def approve_invite(
|
||||
link, has_account, hours = _welcome_mail_parts(db, inv.email)
|
||||
lang = mailtext.lang_for_email(db, inv.email)
|
||||
background.add_task(email_mod.send_access_approved, inv.email, lang, link, has_account, hours)
|
||||
# Before the response, so the mail (a background task, after it) never describes a decision
|
||||
# the database does not hold yet.
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [inv.email])))
|
||||
|
||||
|
||||
@@ -226,6 +230,7 @@ def deny_invite(
|
||||
background.add_task(
|
||||
email_mod.send_access_denied, inv.email, mailtext.lang_for_email(db, inv.email)
|
||||
)
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [inv.email])))
|
||||
|
||||
|
||||
@@ -242,7 +247,7 @@ def add_invite(
|
||||
"""Manually whitelist an email (approved straight away). Convenience for the admin."""
|
||||
email = body.email.strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
# Reuse a canonically-equal row instead of adding a second one for the same mailbox. Prod carries
|
||||
# exactly that pair for one person from 2026-06-16 (hartmann.peter1986@ / hartmannpeter1986@),
|
||||
# which is how an admin ends up approving the same human twice and still seeing them bounce.
|
||||
@@ -259,11 +264,13 @@ def add_invite(
|
||||
inv.status = "approved"
|
||||
inv.decided_at = datetime.now(timezone.utc)
|
||||
inv.decided_by = admin.email
|
||||
db.flush() # a new invite's id, for the audit row — before it, the row named no target
|
||||
audit.record(
|
||||
db, admin.id, AuditAction.INVITE_ADD, target_type="invite", target_id=inv.id, summary=email
|
||||
)
|
||||
db.commit()
|
||||
# One transaction: the whitelisting and the activation it implies land together or not at all.
|
||||
_activate_user_by_email(db, email)
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [email])))
|
||||
|
||||
|
||||
@@ -366,10 +373,7 @@ def delete_invite(
|
||||
promoted to admin on its next Google sign-in). The admin has to hear that, not guess it."""
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
if _accounts_for(db, [inv.email]):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="That address already has an account — delete the account instead.",
|
||||
)
|
||||
raise refused(409, "address_has_account")
|
||||
still_allowed_by_env = _env_allowed(inv.email)
|
||||
audit.record(
|
||||
db,
|
||||
@@ -435,19 +439,19 @@ def set_user_role(
|
||||
account, and can't remove the last remaining admin. Emails the user when it actually changes."""
|
||||
role = body.role
|
||||
if role not in ("user", "admin"):
|
||||
raise HTTPException(status_code=400, detail="role must be 'user' or 'admin'")
|
||||
raise refused(400, "invalid_role")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account's role can't be changed.")
|
||||
raise refused(400, "demo_role")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="You can't change your own role.")
|
||||
raise refused(400, "own_role")
|
||||
if (
|
||||
target.role == "admin"
|
||||
and role == "user"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't remove the last admin.")
|
||||
raise refused(400, "last_admin")
|
||||
changed = target.role != role
|
||||
if changed:
|
||||
audit.record(
|
||||
@@ -493,16 +497,16 @@ def set_user_suspended(
|
||||
suspended = body.suspended
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be suspended.")
|
||||
raise refused(400, "demo_suspend")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="You can't suspend your own account.")
|
||||
raise refused(400, "own_suspend")
|
||||
if (
|
||||
suspended
|
||||
and target.role == "admin"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't suspend the last active admin.")
|
||||
raise refused(400, "last_active_admin")
|
||||
was_suspended = target.is_suspended
|
||||
if was_suspended != suspended:
|
||||
audit.record(
|
||||
@@ -537,17 +541,15 @@ def admin_delete_user(
|
||||
not the demo account, not yourself (use Settings → Account), and not the last admin."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be deleted.")
|
||||
raise refused(400, "demo_delete")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Delete your own account from Settings → Account."
|
||||
)
|
||||
raise refused(400, "own_delete")
|
||||
if (
|
||||
target.role == "admin"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't delete the last admin.")
|
||||
raise refused(400, "last_admin")
|
||||
# Record BEFORE the purge — the target row (and its email) is gone afterwards.
|
||||
audit.record(
|
||||
db,
|
||||
@@ -691,7 +693,7 @@ def activate_user(
|
||||
`_activate_user_by_email`). What was missing was any way to know they had to."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be activated.")
|
||||
raise refused(400, "demo_activate")
|
||||
if target.is_active:
|
||||
return _serialize_user(target) # idempotent: a double click is not an error
|
||||
if not is_allowed(db, target.email):
|
||||
@@ -788,7 +790,7 @@ def add_demo_whitelist(
|
||||
sign-in). Idempotent — re-adding an existing email just returns it."""
|
||||
email = body.email.strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
row = db.execute(select(DemoWhitelist).where(DemoWhitelist.email == email)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = DemoWhitelist(
|
||||
@@ -835,7 +837,7 @@ def reset_demo(admin: User = Depends(admin_user), db: Session = Depends(get_db))
|
||||
db, admin.id, AuditAction.DEMO_RESET, summary=f"{seeded} sample playlists re-seeded"
|
||||
)
|
||||
db.commit()
|
||||
return {"reset": True, "playlists_seeded": seeded}
|
||||
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
||||
|
||||
|
||||
@router.post("/purge-discovery")
|
||||
@@ -854,4 +856,4 @@ def purge_discovery(admin: User = Depends(admin_user), db: Session = Depends(get
|
||||
after=result,
|
||||
)
|
||||
db.commit()
|
||||
return result
|
||||
return {"ok": True, **result}
|
||||
|
||||
@@ -26,10 +26,11 @@ from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import Text, and_, cast, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logsink
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import LogEvent, User
|
||||
from app.utils import iso, like_escape
|
||||
from app.utils import like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/logs", tags=["admin"])
|
||||
|
||||
@@ -66,25 +67,6 @@ def _decode_cursor(cursor: str) -> tuple[datetime, int]:
|
||||
raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc
|
||||
|
||||
|
||||
def _serialize(row: LogEvent) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
"ts": iso(row.ts),
|
||||
"level": row.level,
|
||||
"source": row.source,
|
||||
"logger": row.logger,
|
||||
"event": row.event,
|
||||
"msg": row.msg,
|
||||
"attrs": row.attrs,
|
||||
"exc": row.exc,
|
||||
"request_id": row.request_id,
|
||||
"user_id": row.user_id,
|
||||
"job_id": row.job_id,
|
||||
"debug_session_id": row.debug_session_id,
|
||||
"version": row.version,
|
||||
}
|
||||
|
||||
|
||||
def _accounts(db: Session, user_ids: set[int]) -> dict[str, str]:
|
||||
"""The accounts a page names, by id → address. An id with no account left (erased after the
|
||||
line was written) is simply absent, and the client shows the bare id."""
|
||||
@@ -196,7 +178,7 @@ def list_logs(
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return {
|
||||
"items": [_serialize(r) for r in rows],
|
||||
"items": [logsink.serialize(r) for r in rows],
|
||||
"accounts": _accounts(db, {r.user_id for r in rows if r.user_id is not None}),
|
||||
"next_cursor": _encode_cursor(rows[-1]) if more else None,
|
||||
}
|
||||
|
||||
@@ -68,4 +68,4 @@ def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
n = db.execute(delete(AuditLog)).rowcount or 0
|
||||
audit.record(db, admin.id, AuditAction.AUDIT_CLEAR, summary=f"Cleared {n} audit entries")
|
||||
db.commit()
|
||||
return {"cleared": n}
|
||||
return {"ok": True, "cleared": n}
|
||||
|
||||
+23
-28
@@ -39,10 +39,11 @@ from app.auth import (
|
||||
)
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import AuthToken, DemoWhitelist, Invite, OAuthToken, User
|
||||
from app.models import AuthToken, DemoWhitelist, Invite, OAuthToken, User, display_text
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import encrypt, hash_password, hash_token, verify_password
|
||||
from app.utils import canonical_email, valid_email
|
||||
from app.refusal import refused
|
||||
|
||||
VERIFY_TTL = timedelta(hours=24)
|
||||
# A confirmation link that rides an APPROVAL or an invitation lives longer than one the person is
|
||||
@@ -378,7 +379,7 @@ async def callback(
|
||||
|
||||
userinfo = token.get("userinfo")
|
||||
if not userinfo or not userinfo.get("sub"):
|
||||
raise HTTPException(status_code=400, detail="No user info returned by Google")
|
||||
raise refused(400, "google_no_userinfo")
|
||||
|
||||
sub = userinfo["sub"]
|
||||
email = (userinfo.get("email") or "").lower()
|
||||
@@ -471,7 +472,7 @@ async def callback(
|
||||
if email:
|
||||
user.google_email = email
|
||||
_sync_google_email(db, user, email)
|
||||
user.display_name = userinfo.get("name")
|
||||
user.display_name = display_text(userinfo.get("name"))
|
||||
user.avatar_url = userinfo.get("picture")
|
||||
# ADMIN_EMAILS (env) is the bootstrap admin list and always wins, so the configured admin can't
|
||||
# be locked out. Everyone else's role is managed in the admin UI and stored in the DB, so a
|
||||
@@ -593,7 +594,7 @@ def _complete_link(
|
||||
if linked_email:
|
||||
user.google_email = linked_email # see the sign-in callback: the other half of the identity
|
||||
if not user.display_name:
|
||||
user.display_name = userinfo.get("name")
|
||||
user.display_name = display_text(userinfo.get("name"))
|
||||
if not user.avatar_url:
|
||||
user.avatar_url = userinfo.get("picture")
|
||||
_store_token(db, user, token)
|
||||
@@ -653,15 +654,15 @@ async def request_access(
|
||||
endpoint can't be probed for throttling either."""
|
||||
email = (body.email or "").strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
if not _request_access_limiter.allow(client_key(request)):
|
||||
return {"status": "pending"}
|
||||
return {"ok": True, "status": "pending"}
|
||||
if is_allowed(db, email):
|
||||
return {"status": "pending"} # already allowed — don't reveal it
|
||||
return {"ok": True, "status": "pending"} # already allowed — don't reveal it
|
||||
req = upsert_pending_invite(db, email, body.lang)
|
||||
if req.is_new:
|
||||
notify_admins_of_request(db, background, email)
|
||||
return {"status": "pending"}
|
||||
return {"ok": True, "status": "pending"}
|
||||
|
||||
|
||||
@router.post("/demo")
|
||||
@@ -672,7 +673,7 @@ def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -
|
||||
exactly like a non-match (authenticated=false) so the field can't be hammered as an
|
||||
enumeration oracle. On a match it sets the session and the client reloads into the app."""
|
||||
if not _demo_limiter.allow(client_key(request)):
|
||||
return {"authenticated": False}
|
||||
return {"ok": True, "authenticated": False}
|
||||
email = (body.email or "").strip().lower()
|
||||
if valid_email(email) and is_demo_allowed(db, email):
|
||||
demo = get_or_create_demo_user(db)
|
||||
@@ -683,8 +684,8 @@ def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -
|
||||
_remember_epoch(request, demo)
|
||||
# The demo key IS an address, so it is not in the line — see the denied branch above.
|
||||
log.info("signed in", extra=logs.event("auth.login.ok", method="demo", uid=demo.id))
|
||||
return {"authenticated": True}
|
||||
return {"authenticated": False}
|
||||
return {"ok": True, "authenticated": True}
|
||||
return {"ok": True, "authenticated": False}
|
||||
|
||||
|
||||
def _establish_session(request: Request, user: User) -> None:
|
||||
@@ -815,23 +816,20 @@ def register(
|
||||
password = body.password or ""
|
||||
# Input validation is safe to reveal (independent of whether the email exists).
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
if len(password) < PASSWORD_MIN_LEN:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Password must be at least {PASSWORD_MIN_LEN} characters.",
|
||||
)
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
if not sysconfig.get_bool(db, "allow_registration"):
|
||||
raise HTTPException(status_code=403, detail="Registration is currently closed.")
|
||||
raise refused(403, "registration_closed")
|
||||
if not _register_limiter.allow(client_key(request)):
|
||||
return {"status": "ok"} # silently throttle; uniform response
|
||||
return {"ok": True, "status": "ok"} # silently throttle; uniform response
|
||||
|
||||
# Do the existence check + account creation OFF the response path (a background task with its own
|
||||
# session). The create path hashes the password + writes several rows + schedules emails; doing it
|
||||
# inline would make a NEW email respond measurably slower than an already-registered one (which
|
||||
# skips all that) — a timing oracle for enumeration. Off-path, any valid email responds the same.
|
||||
background.add_task(_register_account, email, password, body.lang)
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
|
||||
|
||||
def _register_account(email: str, password: str, lang: str | None = None) -> None:
|
||||
@@ -997,11 +995,11 @@ def verify_email_resend(
|
||||
mail had no way back. Uniform response + timing regardless of whether the email has a pending
|
||||
account (the lookup runs off the response path), so it can't probe for registered emails."""
|
||||
if not _verify_resend_limiter.allow(client_key(request)):
|
||||
return {"status": "ok"} # silently throttle; uniform response
|
||||
return {"ok": True, "status": "ok"} # silently throttle; uniform response
|
||||
email = (body.email or "").strip().lower()
|
||||
if valid_email(email):
|
||||
background.add_task(_resend_verify_if_eligible, email)
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
|
||||
|
||||
def _resend_verify_if_eligible(email: str) -> None:
|
||||
@@ -1091,14 +1089,14 @@ def password_reset_request(
|
||||
"""Request a password-reset link. Uniform response + timing regardless of whether the email has a
|
||||
password account (the lookup runs off the response path), so it can't probe for registered emails."""
|
||||
if not _reset_limiter.allow(client_key(request)):
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
email = (body.email or "").strip().lower()
|
||||
# Do the account lookup + token issue + email OFF the response path (a background task with its
|
||||
# own session), so the endpoint takes the same time for any valid email whether or not it has a
|
||||
# password account — no timing-based enumeration. Any valid email schedules the same task.
|
||||
if valid_email(email):
|
||||
background.add_task(_send_reset_if_eligible, email)
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
|
||||
|
||||
def _send_reset_if_eligible(email: str) -> None:
|
||||
@@ -1293,13 +1291,10 @@ def set_password(
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
new_password = body.password or ""
|
||||
if len(new_password) < PASSWORD_MIN_LEN:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Password must be at least {PASSWORD_MIN_LEN} characters.",
|
||||
)
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
if user.password_hash:
|
||||
if not verify_password(body.current_password or "", user.password_hash):
|
||||
raise HTTPException(status_code=403, detail="Current password is incorrect.")
|
||||
raise refused(403, "wrong_password")
|
||||
user.password_hash = hash_password(new_password)
|
||||
user.password_revoked_at = None # same as the reset path: a password exists again
|
||||
# SA4: a password change logs out every OTHER session; re-stamp this cookie at the new epoch so
|
||||
|
||||
@@ -4,7 +4,7 @@ hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side w
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -25,12 +25,13 @@ from app.models import (
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.sync.explore import explore_ingest_page
|
||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.sync.subscriptions import apply_channel_details, details_stale
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/channels", tags=["channels"])
|
||||
|
||||
@@ -189,7 +190,7 @@ def discover_channels(
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
apply_channel_details(db, yt.get_channels(need))
|
||||
apply_channel_details(db, yt.get_channels(need), need)
|
||||
db.commit()
|
||||
# Match _discovery_rows' ORDER BY exactly: video-count DESC, then title (NULLs LAST,
|
||||
# as Postgres orders them — coercing NULL→"" would wrongly float untitled channels up).
|
||||
@@ -295,17 +296,20 @@ def channel_detail(
|
||||
"""One channel's "About" detail for the channel page header/About tab, plus this user's
|
||||
relationship to it (subscribed / explored) and how many of its videos are in the catalog.
|
||||
Lazily fills the About metadata (view count, join date, banner, links) the first time it's
|
||||
needed — a stub, or a channel last enriched before those fields existed (published_at is a
|
||||
reliable sentinel: every real channel has a creation date). The enrich uses the API key
|
||||
(1 unit, cached after), so it's skipped for the quota-less demo account."""
|
||||
needed (a stub) and refreshes it once it is stale (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the
|
||||
subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit,
|
||||
cached after), so it's skipped for the quota-less demo account."""
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if channel.published_at is None and not user.is_demo:
|
||||
# No `published_at is None` sentinel: a terminated channel never gets one, so it re-fetched on
|
||||
# every view; rows enriched before that field existed are long past DETAILS_MAX_AGE anyway.
|
||||
stale = details_stale(channel, datetime.now(timezone.utc))
|
||||
if stale and not user.is_demo:
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||
db.commit()
|
||||
channel = db.get(Channel, channel_id)
|
||||
except YouTubeError as exc:
|
||||
@@ -383,12 +387,9 @@ def explore_channel(
|
||||
backfill quota reserve; blocked for the demo account (it spends shared quota)."""
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if _is_blocked(db, user, channel_id):
|
||||
raise HTTPException(status_code=403, detail="You've blocked this channel.")
|
||||
raise refused(403, "channel_blocked")
|
||||
if quota.backfill_reserve_reached():
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="The shared daily quota reserve is reached — try exploring later.",
|
||||
)
|
||||
raise refused(429, "quota_reserve")
|
||||
|
||||
# Record/refresh the exploration (DB only) before ingesting, so the explorer can see the
|
||||
# videos as soon as they land. Flag the channel ephemeral unless it's organically followed.
|
||||
@@ -414,18 +415,17 @@ def explore_channel(
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
# Enrich for the About panel + the uploads playlist id (needed to ingest videos).
|
||||
if (
|
||||
channel.details_synced_at is None
|
||||
or channel.uploads_playlist_id is None
|
||||
or channel.published_at is None
|
||||
):
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
# Stale, not "a field is missing": a terminated channel never gets an uploads playlist
|
||||
# or a creation date, and asking again on every explore would spend quota on a miss.
|
||||
if details_stale(channel, datetime.now(timezone.utc)):
|
||||
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||
db.commit()
|
||||
channel = db.get(Channel, channel_id)
|
||||
result = explore_ingest_page(db, yt, channel, page_token)
|
||||
except YouTubeError as exc:
|
||||
raise HTTPException(status_code=422, detail=f"YouTube couldn't load this channel: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
return {
|
||||
"ok": True,
|
||||
"channel_id": channel_id,
|
||||
"ingested": result["ingested"],
|
||||
"next_page_token": result["next_page_token"],
|
||||
@@ -507,6 +507,7 @@ def update_channel(
|
||||
background.add_task(backfill_channel_now, channel_id, user.id)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"id": channel_id,
|
||||
"priority": sub.priority,
|
||||
"hidden": sub.hidden,
|
||||
@@ -541,7 +542,7 @@ def reset_backfill(
|
||||
db.commit()
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
||||
run_recent_backfill(db, [channel], max_channels=1)
|
||||
return {"id": channel_id, "reset": True}
|
||||
return {"ok": True, "id": channel_id, "reset": True}
|
||||
|
||||
|
||||
@router.post("/{channel_id}/subscribe")
|
||||
@@ -562,7 +563,7 @@ def subscribe(
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise HTTPException(status_code=409, detail="Already subscribed to this channel.")
|
||||
raise refused(409, "already_subscribed")
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE),
|
||||
YouTubeClient(db, user) as yt,
|
||||
@@ -583,16 +584,14 @@ def subscribe(
|
||||
if "subscriptionDuplicate" in exc.reason.split(","):
|
||||
yt_sub_id = ""
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"YouTube couldn't subscribe you: {exc}"
|
||||
)
|
||||
raise youtube_refusal(exc)
|
||||
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
|
||||
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
|
||||
# render. Keep the client open (we're still inside the `with`); a metadata failure here
|
||||
# must not block recording the subscription the user asked for.
|
||||
if channel.details_synced_at is None:
|
||||
try:
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||
except YouTubeError as exc:
|
||||
log.warning(
|
||||
"subscribe: stub enrich failed: %s",
|
||||
@@ -622,7 +621,7 @@ def subscribe(
|
||||
.values(via_explore=False)
|
||||
)
|
||||
db.commit()
|
||||
return {"subscribed": channel_id}
|
||||
return {"ok": True, "subscribed": channel_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/subscription")
|
||||
@@ -636,10 +635,7 @@ def unsubscribe(
|
||||
should Hide instead."""
|
||||
sub = _user_subscription(db, user, channel_id)
|
||||
if not sub.yt_subscription_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="No YouTube subscription id on record — sync subscriptions first.",
|
||||
)
|
||||
raise refused(400, "no_subscription_id")
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_UNSUBSCRIBE),
|
||||
@@ -649,10 +645,10 @@ def unsubscribe(
|
||||
except YouTubeError as exc:
|
||||
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
|
||||
# a transient "connection lost" gateway blip.
|
||||
raise HTTPException(status_code=422, detail=f"YouTube couldn't unsubscribe you: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
db.delete(sub)
|
||||
db.commit()
|
||||
return {"unsubscribed": channel_id}
|
||||
return {"ok": True, "unsubscribed": channel_id}
|
||||
|
||||
|
||||
@router.get("/blocked/list")
|
||||
@@ -693,7 +689,7 @@ def block_channel(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"blocked": channel_id}
|
||||
return {"ok": True, "blocked": channel_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/block")
|
||||
@@ -708,7 +704,7 @@ def unblock_channel(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"unblocked": channel_id}
|
||||
return {"ok": True, "unblocked": channel_id}
|
||||
|
||||
|
||||
class AttachTagIn(BaseModel):
|
||||
@@ -743,7 +739,7 @@ def attach_tag(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"channel_id": channel_id, "tag_id": body.tag_id}
|
||||
return {"ok": True, "channel_id": channel_id, "tag_id": body.tag_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/tags/{tag_id}")
|
||||
@@ -763,4 +759,4 @@ def detach_tag(
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"channel_id": channel_id, "tag_id": tag_id}
|
||||
return {"ok": True, "channel_id": channel_id, "tag_id": tag_id}
|
||||
|
||||
@@ -4,7 +4,7 @@ app.sysconfig. Secret values (e.g. SMTP password) are write-only — never retur
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -17,6 +17,10 @@ from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import User
|
||||
from app.refusal import refused
|
||||
from app.issues import gitea
|
||||
from app.routes._common import issue_refused, not_found
|
||||
from app.refusal import setting_refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin/config", tags=["admin"])
|
||||
|
||||
@@ -59,19 +63,16 @@ def set_config(
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown config key")
|
||||
raise not_found("config_key")
|
||||
if "value" not in body.model_fields_set:
|
||||
raise HTTPException(status_code=400, detail="Missing 'value'")
|
||||
raise refused(400, "value_missing")
|
||||
if s.secret and not sysconfig.secrets_manageable():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Set TOKEN_ENCRYPTION_KEY to store secrets in the database.",
|
||||
)
|
||||
raise refused(400, "no_encryption_key")
|
||||
old = None if s.secret else sysconfig.get(db, key)
|
||||
try:
|
||||
sysconfig.set_value(db, key, body.value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc) or "Invalid value")
|
||||
except sysconfig.InvalidSetting as exc:
|
||||
raise setting_refused(exc) from None
|
||||
# Never log a secret's value — only that it was changed, by whom. Non-secret values are logged
|
||||
# but with URL credentials redacted (a proxy/URL setting can embed user:pass). Skip the row when
|
||||
# a non-secret value didn't actually change (a no-op re-save shouldn't add trail noise); secrets
|
||||
@@ -101,7 +102,7 @@ def reset_config(
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown config key")
|
||||
raise not_found("config_key")
|
||||
old = None if s.secret else sysconfig.get(db, key)
|
||||
sysconfig.reset(db, key)
|
||||
audit.record(
|
||||
@@ -126,8 +127,21 @@ def send_test_email(
|
||||
"""Send a test email to the requesting admin using the current (DB or env) SMTP config —
|
||||
lets the admin verify the settings actually work."""
|
||||
if not email_mod.email_enabled():
|
||||
raise HTTPException(status_code=400, detail="SMTP is not configured.")
|
||||
raise refused(400, "smtp_off")
|
||||
sent = email_mod.send_test(user.email, mailtext.lang_for_user(user))
|
||||
if not sent:
|
||||
raise HTTPException(status_code=422, detail="SMTP send failed — check the settings.")
|
||||
return {"sent": True, "to": user.email}
|
||||
raise refused(422, "smtp_failed")
|
||||
return {"ok": True, "sent": True, "to": user.email}
|
||||
|
||||
|
||||
@router.post("/test-issues")
|
||||
def test_issue_tracker(
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Check the configured issue tracker (E16): the repository answers with this token, takes
|
||||
issues, and is PRIVATE — the one it must be, since an issue carries its reporter's log."""
|
||||
try:
|
||||
return {"ok": True, **gitea.check(db)}
|
||||
except gitea.GiteaError as exc:
|
||||
raise issue_refused(exc) from None
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, the screenshots, and the evidence under review.
|
||||
|
||||
The session itself — what tags a request, its cap, its level — is `app/debugsession.py`. These routes
|
||||
are the user's handle on their OWN sessions only: another user's id 404s (`owned_or_404`), including
|
||||
for an admin, who reads everyone's lines in the diagnostics console instead. The shared demo account
|
||||
cannot record: its log would interleave every visitor under one account.
|
||||
|
||||
Screenshots arrive as a raw image body (`image/png`, `image/jpeg`, `image/webp`), not a multipart
|
||||
form — one image per request, capped while it arrives, and kept only if its bytes really are one of
|
||||
those types, whatever the Content-Type claimed.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, debugsession, logs, logsink
|
||||
from app.audit import AuditAction
|
||||
from app.auth import require_human
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.issues import bundle, gitea
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.refusal import refused
|
||||
from app.models import DebugScreenshot, DebugSession, User
|
||||
from app.routes._common import capped_body, issue_refused, not_found, owned_or_404
|
||||
from app.utils import iso
|
||||
|
||||
router = APIRouter(prefix="/api/debug-sessions", tags=["diagnostics"])
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Sessions listed in Settings → Diagnosztika, newest first: enough to reach last week's unsent draft.
|
||||
LIST_MAX = 20
|
||||
|
||||
|
||||
class StartIn(BaseModel):
|
||||
level: debugsession.SessionLevel
|
||||
|
||||
|
||||
_view = debugsession.view
|
||||
|
||||
# New sessions a user may open per hour (joining the running one is not counted): a session is a
|
||||
# place to keep screenshots, and this — with the per-user storage cap — bounds how many there are.
|
||||
_new_sessions = RateLimiter(max_events=20, window_seconds=3600)
|
||||
|
||||
|
||||
@router.post("")
|
||||
def start_session(
|
||||
body: StartIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Start recording, or join the recording already running (another tab started it)."""
|
||||
joining = debugsession.open_session(db, user) is not None
|
||||
if not joining and not _new_sessions.allow(str(user.id)):
|
||||
raise refused(429, "debug_session_rate_limited", limit=_new_sessions.max_events)
|
||||
try:
|
||||
session = debugsession.start(db, user, body.level)
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
# A concurrent Start won the one-open-per-user index: join that one.
|
||||
db.rollback()
|
||||
session = debugsession.open_session(db, user)
|
||||
if session is None:
|
||||
raise refused(409, "debug_session_conflict") from None
|
||||
# The rest of THIS request is part of the session too.
|
||||
level, until = debugsession.level_for(user, session)
|
||||
logs.set_user(user.id, level, until, session.id)
|
||||
log.info(
|
||||
"debug session started",
|
||||
extra=logs.event("debug.session.started", level=session.level, until=iso(session.until)),
|
||||
)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.post("/{sid}/stop")
|
||||
def stop_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if session.ended_at is None:
|
||||
# Logged BEFORE the session closes, while this request still carries its tag: the last line
|
||||
# of the evidence is the Stop itself.
|
||||
log.info("debug session stopped", extra=logs.event("debug.session.stopped"))
|
||||
debugsession.stop(session)
|
||||
db.commit()
|
||||
logs.end_session_elevation(session.id)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_sessions(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
sessions = db.scalars(
|
||||
select(DebugSession)
|
||||
.where(DebugSession.user_id == user.id)
|
||||
.order_by(DebugSession.started_at.desc())
|
||||
.limit(LIST_MAX)
|
||||
).all()
|
||||
return {"items": [_view(db, s) for s in sessions]}
|
||||
|
||||
|
||||
@router.get("/{sid}")
|
||||
def get_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
return _view(db, owned_or_404(db, DebugSession, sid, user, what="debug_session"))
|
||||
|
||||
|
||||
@router.get("/{sid}/events")
|
||||
def session_events(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""The rows an issue from this session would carry — browser and server, in time order."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
rows, total = debugsession.bundle(db, session)
|
||||
return {"items": [logsink.serialize(r) for r in rows], "total": total}
|
||||
|
||||
|
||||
# --- the draft and its export ---------------------------------------------------------------
|
||||
# The composer's own fields (steps, description…) are its business, kept whole so it reopens as it
|
||||
# was — bounded here so a draft cannot become a place to store anything else.
|
||||
FORM_MAX_BYTES = 64 * 1024
|
||||
# Zips a user may build per minute: each holds every screenshot in memory while it is built.
|
||||
_exports = RateLimiter(max_events=6, window_seconds=60)
|
||||
|
||||
|
||||
class DraftIn(BaseModel):
|
||||
title: str = Field(max_length=200)
|
||||
# The rendered markdown the export and the issue carry.
|
||||
body: str = Field(max_length=50_000)
|
||||
# Log rows taken out of the evidence in the review.
|
||||
excluded: list[int] = Field(default_factory=list, max_length=debugsession.BUNDLE_MAX)
|
||||
form: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@router.put("/{sid}/draft")
|
||||
def save_draft(
|
||||
sid: str,
|
||||
body: DraftIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if len(json.dumps(body.form, ensure_ascii=False).encode()) > FORM_MAX_BYTES:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
# Under the lock, and refused while a send is under way: the issue is being built from the draft
|
||||
# as it was saved, and this would change it under the send.
|
||||
debugsession.lock(db, session)
|
||||
if debugsession.sending(session):
|
||||
raise refused(409, "issue_sending")
|
||||
if session.issue_url is not None:
|
||||
# Sent: the issue is on the tracker now, and a draft edited after it would say otherwise.
|
||||
raise refused(409, "issue_already_sent")
|
||||
debugsession.save_draft(session, body.model_dump())
|
||||
db.commit()
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.get("/{sid}/export")
|
||||
def export_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
"""The issue as a zip — the text, the two logs and the screenshots — for an instance without an
|
||||
issue tracker, or for sending it by hand. Built from the SAVED draft: what the composer shows is
|
||||
what it saved."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if not _exports.allow(str(user.id)):
|
||||
raise refused(429, "export_rate_limited")
|
||||
data = bundle.zipped(bundle.files(db, session))
|
||||
name = f"siftlode-debug-{session.started_at:%Y%m%d-%H%M}-{session.id[:6]}.zip"
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="application/zip",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{name}"',
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# --- the issue tracker (S4) -----------------------------------------------------------------
|
||||
# Issues a user may open per hour — counted once the tracker is ASKED to open one (an answer that
|
||||
# goes missing may still have created it, so an attempt cannot be given back); finishing a half-sent
|
||||
# one, or an attempt refused before the tracker was asked, is not counted. The composer sends one at
|
||||
# a time; this is room for a bad afternoon, not a pace anything legitimate keeps up.
|
||||
_new_issues = RateLimiter(max_events=5, window_seconds=3600)
|
||||
# Attempts on the tracker per hour, creating or finishing. Deliberately NOT the budget above: a
|
||||
# half-sent issue must always be finishable, so the pace that guards creating one must never be what
|
||||
# stands between a user and their own report. This one only bounds a loop — every attempt re-reads
|
||||
# the whole bundle into memory and re-offers the pending file to the tracker, and a retry that is
|
||||
# refused for good (the tracker's own limits) never leaves `pending`. No hand reaches it.
|
||||
_sends = RateLimiter(max_events=60, window_seconds=3600)
|
||||
# The composer's live preview, rendered BY the tracker: one authenticated call each, with our token.
|
||||
# The composer's toggle fires it once per switch INTO preview — the editor is the other branch, so
|
||||
# no keystroke can change the query key while the preview is up — and react-query's cache makes an
|
||||
# unchanged draft free. (It does NOT debounce, whatever this comment used to say: a key that changes
|
||||
# per keystroke is one fetch per keystroke, measured. Showing the preview BESIDE the fields would
|
||||
# need a real debounce first.) This bounds what a loop can aim at the tracker.
|
||||
_previews = RateLimiter(max_events=60, window_seconds=60)
|
||||
|
||||
|
||||
def _issues_rate_limited() -> HTTPException:
|
||||
return refused(429, "issues_rate_limited", limit=_new_issues.max_events)
|
||||
|
||||
|
||||
@router.post("/{sid}/issue")
|
||||
def send_issue(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Send the session's issue to the tracker — from the SAVED draft, as the export is built — or
|
||||
finish a send that stopped half-way (the issue exists; only what is missing is sent)."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
# The session is CLAIMED for the send rather than locked through it: the tracker is slow (a
|
||||
# dozen calls, 20 s each), and a row lock held across them would queue this user's Stop, their
|
||||
# autosave and their screenshots behind it. They see the claim instead and are refused at once.
|
||||
debugsession.lock(db, session)
|
||||
if debugsession.sending(session):
|
||||
raise refused(409, "issue_sending")
|
||||
new = session.issue_url is None
|
||||
if not new and not debugsession.unfinished(session):
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
key = str(user.id)
|
||||
if new:
|
||||
if not ((session.draft or {}).get("title") or "").strip():
|
||||
raise refused(422, "issue_title_required")
|
||||
if _new_issues.exhausted(key):
|
||||
raise _issues_rate_limited()
|
||||
if not _sends.allow(key):
|
||||
raise refused(429, "issue_rate_limited", limit=_sends.max_events)
|
||||
token = debugsession.claim_send(session)
|
||||
db.commit() # the claim is what everything else sees; the lock ends here
|
||||
|
||||
def count_new_issue() -> None:
|
||||
if not _new_issues.allow(key):
|
||||
raise _issues_rate_limited()
|
||||
|
||||
try:
|
||||
gitea.deliver(
|
||||
db,
|
||||
session,
|
||||
token=token,
|
||||
checkpoint=db.commit,
|
||||
before_create=count_new_issue,
|
||||
on_created=lambda: _record_created(db, user, session),
|
||||
)
|
||||
except debugsession.SendClaimLost:
|
||||
# This send outlived its lease and another one took the session over while it was still
|
||||
# talking to the tracker. It stopped rather than writing over that one's progress — and a
|
||||
# send IS under way, just not ours, which is what the caller needs to hear.
|
||||
raise refused(409, "issue_sending") from None
|
||||
except gitea.GiteaError as exc:
|
||||
# Keep what did happen: an issue created before an upload failed stays linked, and the
|
||||
# retry does only what is still missing.
|
||||
raise issue_refused(exc) from None
|
||||
except HTTPException:
|
||||
raise # the pace, refused before the tracker was asked
|
||||
finally:
|
||||
_end_send(db, session, token)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
def _end_send(db: Session, session: DebugSession, token: str) -> None:
|
||||
"""Give the claim back, whatever else happened. A failure here must not replace the failure that
|
||||
brought us: a dead transaction (the database went away mid-send) is rolled back, and the claim
|
||||
then simply ages out."""
|
||||
try:
|
||||
debugsession.lock(db, session)
|
||||
debugsession.release_send(session, token)
|
||||
db.commit()
|
||||
except Exception: # noqa: BLE001 — whatever went wrong here, the send's own failure is the news
|
||||
db.rollback()
|
||||
log.warning(
|
||||
"a send could not give its claim back",
|
||||
extra=logs.event("issues.claim.stuck", session_id=session.id),
|
||||
)
|
||||
|
||||
|
||||
def _record_created(db: Session, user: User, session: DebugSession) -> None:
|
||||
"""The trail for the one act that takes a user's log off this instance. Staged inside the send,
|
||||
so the checkpoint that first writes `issue_url` commits BOTH — recorded later it could be lost
|
||||
on its own (the process dies mid-send), and no retry could write it afterwards: the next send
|
||||
sees the link and knows the issue is not new."""
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
AuditAction.ISSUE_CREATE,
|
||||
target_type="debug_session",
|
||||
target_id=session.id,
|
||||
summary=session.issue_url or "",
|
||||
)
|
||||
|
||||
|
||||
class PreviewIn(BaseModel):
|
||||
text: str = Field(max_length=50_000)
|
||||
|
||||
|
||||
@router.post("/{sid}/preview")
|
||||
def preview_issue(
|
||||
sid: str,
|
||||
body: PreviewIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The body as the tracker will render it. Gitea sanitises its own output; the composer still
|
||||
shows it in a sandboxed frame, never in the app's own document."""
|
||||
owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if not _previews.allow(str(user.id)):
|
||||
raise refused(429, "preview_rate_limited")
|
||||
try:
|
||||
return {"ok": True, "html": gitea.preview(db, body.text)}
|
||||
except gitea.GiteaError as exc:
|
||||
raise issue_refused(exc) from None
|
||||
|
||||
|
||||
# --- screenshots ---------------------------------------------------------------------------
|
||||
def _shot_view(shot: DebugScreenshot) -> dict:
|
||||
return {"id": shot.id, "mime": shot.mime, "size": shot.size, "created_at": iso(shot.created_at)}
|
||||
|
||||
|
||||
@router.get("/{sid}/screenshots")
|
||||
def list_screenshots(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
return {"items": [_shot_view(s) for s in debugsession.screenshots(db, session)]}
|
||||
|
||||
|
||||
def _upload_target(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> DebugSession:
|
||||
return owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
|
||||
|
||||
async def _image_body(request: Request, db: Session = Depends(get_db)) -> bytes:
|
||||
# The owner check above has already run, and it was a SELECT — so this session is holding a
|
||||
# pooled connection `idle in transaction` and would hold it for the whole 10 MiB upload, from a
|
||||
# client that may be on a phone. Hand it back first; the handler's own work re-takes one. The
|
||||
# `session` object the handler holds is expired by this and reloads on its next attribute read,
|
||||
# which is the correct moment to touch the database again.
|
||||
release_read_connection(db)
|
||||
return await capped_body(request, debugsession.SCREENSHOT_MAX_BYTES)
|
||||
|
||||
|
||||
@router.post("/{sid}/screenshots")
|
||||
def add_screenshot(
|
||||
# The owner check FIRST: dependencies resolve in order, so a stranger's upload is refused
|
||||
# before a byte of it is read.
|
||||
session: DebugSession = Depends(_upload_target),
|
||||
data: bytes = Depends(_image_body),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
try:
|
||||
shot = debugsession.add_screenshot(db, session, data)
|
||||
except debugsession.ScreenshotRefused as exc:
|
||||
raise _screenshot_refused(exc.code) from None
|
||||
db.commit()
|
||||
return {"ok": True, "screenshot": _shot_view(shot)}
|
||||
|
||||
|
||||
def _screenshot_refused(code: str) -> HTTPException:
|
||||
# Literal reasons, one per case: the frontend's contract test reads them to require a sentence
|
||||
# for each in both languages.
|
||||
if code == "already_sent":
|
||||
return refused(409, "screenshot_already_sent")
|
||||
if code == "too_many":
|
||||
return refused(422, "screenshot_too_many", limit=debugsession.SCREENSHOTS_MAX)
|
||||
if code == "sending":
|
||||
return refused(409, "issue_sending")
|
||||
if code == "storage_full":
|
||||
mb = debugsession.SCREENSHOT_USER_MAX_BYTES // (1024 * 1024)
|
||||
return refused(422, "screenshot_storage_full", limit=mb)
|
||||
if code == "empty":
|
||||
return refused(422, "screenshot_empty")
|
||||
return refused(422, "screenshot_not_an_image")
|
||||
|
||||
|
||||
def _session_screenshot(db: Session, sid: str, shot_id: int, user: User) -> DebugScreenshot:
|
||||
return _screenshot_of(
|
||||
db, owned_or_404(db, DebugSession, sid, user, what="debug_session"), shot_id
|
||||
)
|
||||
|
||||
|
||||
def _screenshot_of(db: Session, session: DebugSession, shot_id: int) -> DebugScreenshot:
|
||||
shot = db.get(DebugScreenshot, shot_id)
|
||||
if shot is None or shot.session_id != session.id:
|
||||
raise not_found("screenshot")
|
||||
return shot
|
||||
|
||||
|
||||
@router.get("/{sid}/screenshots/{shot_id}")
|
||||
def get_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
shot = _session_screenshot(db, sid, shot_id, user)
|
||||
# `nosniff`: the type was checked against the bytes at upload, and the browser must not guess.
|
||||
return Response(
|
||||
content=shot.data,
|
||||
media_type=shot.mime,
|
||||
headers={"Cache-Control": "private, max-age=3600", "X-Content-Type-Options": "nosniff"},
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{sid}/screenshots/{shot_id}")
|
||||
def delete_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
shot = _screenshot_of(db, session, shot_id)
|
||||
try:
|
||||
debugsession.remove_screenshot(db, session, shot)
|
||||
except debugsession.ScreenshotRefused as exc:
|
||||
raise _screenshot_refused(exc.code) from None
|
||||
db.commit()
|
||||
return {"ok": True, "deleted": shot_id}
|
||||
@@ -13,7 +13,9 @@ Nothing from the body is trusted as-is:
|
||||
* everything is redacted AGAIN with the server's own rules — the client redacts at capture, but a
|
||||
modified client would not;
|
||||
* an event below the consented level is dropped (a Részletes consent does not store what only
|
||||
Teljes may carry), and so is the whole batch once the level has lapsed;
|
||||
Teljes may carry), and so is the whole batch once the level has lapsed — where an open debug
|
||||
session the request names counts as consent to ITS level (E16), and only its own events keep
|
||||
its tag;
|
||||
* the timestamp is clamped to the consent window, so a row cannot dodge the retention job by
|
||||
claiming to come from the future.
|
||||
"""
|
||||
@@ -27,11 +29,12 @@ from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy import insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import diagnostics, logs, logsink
|
||||
from app import debugsession, diagnostics, logs, logsink
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import LogEvent, User
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.models import DebugSession, LogEvent, User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import capped_body
|
||||
|
||||
router = APIRouter(prefix="/api/diagnostics", tags=["diagnostics"])
|
||||
|
||||
@@ -60,6 +63,9 @@ class ClientEvent(BaseModel):
|
||||
attrs: dict[str, Any] | None = None
|
||||
request_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$")
|
||||
tab_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,32}$")
|
||||
# The debug session the event was captured in (E16); kept only when it is the one this request
|
||||
# names in `X-Debug-Session` and the auth dependency found it the caller's own and open.
|
||||
debug_session_id: str | None = Field(default=None, pattern=r"^[0-9a-f]{16}$")
|
||||
version: str | None = Field(default=None, max_length=64)
|
||||
|
||||
|
||||
@@ -67,19 +73,7 @@ class ClientEventsIn(BaseModel):
|
||||
events: list[ClientEvent] = Field(max_length=EVENTS_MAX)
|
||||
|
||||
|
||||
async def _capped_body(request: Request) -> bytes:
|
||||
"""The request body, refused the moment it passes BODY_MAX_BYTES while still arriving. Counted,
|
||||
not taken from Content-Length: a declared length can be absent (chunked) or wrong, and counting
|
||||
stops reading at the same point a declared-length check would have."""
|
||||
body = bytearray()
|
||||
async for chunk in request.stream():
|
||||
body += chunk
|
||||
if len(body) > BODY_MAX_BYTES:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
return bytes(body)
|
||||
|
||||
|
||||
async def _parsed_events(request: Request) -> ClientEventsIn:
|
||||
async def _parsed_events(request: Request, db: Session = Depends(get_db)) -> ClientEventsIn:
|
||||
"""The body as the model — parsed only after the size cap, which is why this is a dependency and
|
||||
not a plain body parameter (FastAPI reads a declared body in full before anything else runs).
|
||||
|
||||
@@ -89,14 +83,24 @@ async def _parsed_events(request: Request) -> ClientEventsIn:
|
||||
media_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media_type != "application/json" and not media_type.endswith("+json"):
|
||||
raise HTTPException(status_code=415, detail={"code": "json_required"})
|
||||
raw = await _capped_body(request)
|
||||
# `current_user` resolved above on a SELECT, so this session is holding a pooled connection.
|
||||
# Give it back before awaiting the body — the sink is a background beacon and its sender may be
|
||||
# on a link far slower than the 512 KiB suggests. See `db.release_read_connection`.
|
||||
release_read_connection(db)
|
||||
raw = await capped_body(request, BODY_MAX_BYTES)
|
||||
try:
|
||||
return ClientEventsIn.model_validate_json(raw)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=422, detail=json.loads(exc.json())) from exc
|
||||
|
||||
|
||||
def _row(item: ClientEvent, user: User, now: datetime, window_start: datetime) -> dict[str, Any]:
|
||||
def _row(
|
||||
item: ClientEvent,
|
||||
user: User,
|
||||
now: datetime,
|
||||
window_start: datetime,
|
||||
session: DebugSession | None,
|
||||
) -> dict[str, Any]:
|
||||
attrs = logs.redact(item.attrs) if item.attrs else None
|
||||
if attrs is not None:
|
||||
size = len(json.dumps(attrs, ensure_ascii=False, default=str).encode())
|
||||
@@ -117,6 +121,9 @@ def _row(item: ClientEvent, user: User, now: datetime, window_start: datetime) -
|
||||
"attrs": attrs,
|
||||
"request_id": item.request_id,
|
||||
"user_id": user.id,
|
||||
"debug_session_id": (
|
||||
session.id if session and item.debug_session_id == session.id else None
|
||||
),
|
||||
"version": item.version,
|
||||
}
|
||||
)
|
||||
@@ -126,11 +133,15 @@ def _row(item: ClientEvent, user: User, now: datetime, window_start: datetime) -
|
||||
def ingest_events(
|
||||
# The account first: dependencies resolve in order, so an anonymous caller is refused before a
|
||||
# byte of its body is read.
|
||||
request: Request,
|
||||
user: User = Depends(current_user),
|
||||
body: ClientEventsIn = Depends(_parsed_events),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
level = diagnostics.effective_level(user)
|
||||
# A debug session the request names raises the level for its own events (E16): starting one is
|
||||
# the consent to that level for its duration, whatever the user's own level is.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, _ = debugsession.level_for(user, session)
|
||||
if level == diagnostics.BASE:
|
||||
# Lapsed or withdrawn: the client stops streaming when it sees this.
|
||||
raise HTTPException(status_code=409, detail={"code": "diagnostics_off"})
|
||||
@@ -140,7 +151,7 @@ def ingest_events(
|
||||
window_start = now - timedelta(hours=diagnostics.ELEVATION_HOURS)
|
||||
floor = logs.SEVERITY[level]
|
||||
rows = [
|
||||
_row(item, user, now, window_start)
|
||||
_row(item, user, now, window_start, session)
|
||||
for item in body.events
|
||||
if logs.SEVERITY[item.level] >= floor
|
||||
]
|
||||
|
||||
@@ -11,9 +11,9 @@ thing it exists to remove.
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -23,7 +23,7 @@ from app.discover import hero as hero_mod
|
||||
from app.discover import pins, rails
|
||||
from app.discover.budget import Budget, DEFAULT_BUDGET
|
||||
from app.models import ComfortPin, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.routes._common import owned_or_404, not_found
|
||||
|
||||
router = APIRouter(prefix="/api/discover", tags=["discover"])
|
||||
|
||||
@@ -53,7 +53,9 @@ def get_discover(
|
||||
`resume` and `shelf` are built once and handed to the hero, so the two rails the hero draws from
|
||||
are not queried twice for a single page load."""
|
||||
resume = rails.continue_watching(db, user)
|
||||
shelf = pins.shelf(db, user)
|
||||
# Plex off → the shelf rail (and so the hero's comfort tier) goes YouTube-only like the rails
|
||||
# beside it; the pins are kept and `GET /pins` still lists them (`pins.shelf`).
|
||||
shelf = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf),
|
||||
@@ -125,7 +127,7 @@ def add_pin(
|
||||
constraint is the mechanism, not the message."""
|
||||
resolved = pins.resolve_ref(db, user, body.kind, body.ref)
|
||||
if resolved is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
raise not_found("item")
|
||||
column, value = resolved
|
||||
|
||||
existing = db.execute(
|
||||
@@ -157,7 +159,7 @@ def add_pin(
|
||||
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
||||
# item), which trips the FK. Semantically that is the same "no such thing" the pre-check
|
||||
# returns above — re-raising handed the user a 500 they could do nothing with.
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
raise not_found("item")
|
||||
return {"ok": True, "id": winner.id, "created": False}
|
||||
return {"ok": True, "id": pin.id, "created": True}
|
||||
|
||||
@@ -180,18 +182,11 @@ def reorder_pins(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Rewrite the shelf order from a full list of pin ids.
|
||||
"""Rewrite the shelf order from the list of pin ids the client shows.
|
||||
|
||||
Whole-list rather than a move-one delta: the shelf is short, and a full rewrite cannot leave two
|
||||
tiles claiming the same slot the way an interrupted sequence of single moves can. Ids that are not
|
||||
this user's are ignored rather than 404'd — a stale tab replaying an order that includes a pin
|
||||
deleted meanwhile should reorder what still exists, not fail wholesale."""
|
||||
owned = {p.id for p in pins.list_pins(db, user)}
|
||||
for position, pin_id in enumerate(i for i in body.ids if i in owned):
|
||||
db.execute(
|
||||
update(ComfortPin)
|
||||
.where(ComfortPin.id == pin_id, ComfortPin.user_id == user.id)
|
||||
.values(position=position)
|
||||
)
|
||||
tiles claiming the same slot the way an interrupted sequence of single moves can. The list may be
|
||||
PARTIAL (hidden Plex pins, a stale tab) — `pins.reorder` keeps what it leaves out in place."""
|
||||
pins.reorder(db, user, body.ids)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
+163
-93
@@ -9,11 +9,13 @@ serves finished files (range-aware, with the user's custom display name), and sh
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
@@ -22,7 +24,7 @@ from sqlalchemy.orm import Session
|
||||
from app import audit, sysconfig
|
||||
from app.auth import admin_user, require_human
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.downloads import credentials as dlcreds
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import links as linksmod
|
||||
@@ -37,11 +39,13 @@ from app.models import (
|
||||
DownloadShare,
|
||||
MediaAsset,
|
||||
User,
|
||||
display_text,
|
||||
)
|
||||
from app.routes._common import get_or_404, not_found
|
||||
from app.routes._common import capped_stream_to, get_or_404, not_found, owned_or_404
|
||||
from app.security import hash_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
|
||||
admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"])
|
||||
@@ -63,7 +67,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
as a raw URL for yt-dlp's generic extractor."""
|
||||
raw = (raw or "").strip()
|
||||
if not raw:
|
||||
raise HTTPException(status_code=400, detail="A video link or id is required.")
|
||||
raise refused(400, "link_required")
|
||||
if _VIDEO_ID.match(raw):
|
||||
return "youtube", raw
|
||||
if raw.startswith(("http://", "https://")):
|
||||
@@ -93,7 +97,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
if _VIDEO_ID.match(vid):
|
||||
return "youtube", vid
|
||||
return "url", raw
|
||||
raise HTTPException(status_code=400, detail="That doesn't look like a valid video link.")
|
||||
raise refused(400, "invalid_link")
|
||||
|
||||
|
||||
def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
@@ -166,15 +170,8 @@ def _serialize_job(db: Session, job: DownloadJob) -> dict:
|
||||
return _serialize(job, asset)
|
||||
|
||||
|
||||
def _own_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None or job.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
return job
|
||||
|
||||
|
||||
def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""`_own_job` for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
||||
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
||||
or writes the status those decide on (pause). The job is re-read under a row lock, so two
|
||||
overlapping requests for one job run one after the other and the second decides on the first's
|
||||
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||
@@ -183,7 +180,7 @@ def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||
(`service.finalize_if_running`), so the two can never deadlock."""
|
||||
while True:
|
||||
asset_id = _own_job(db, user, job_id).asset_id
|
||||
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
@@ -193,7 +190,7 @@ def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
raise not_found("download")
|
||||
if job.asset_id == asset_id:
|
||||
return job
|
||||
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||
@@ -240,7 +237,7 @@ def create_profile(
|
||||
name = body.name.strip()
|
||||
spec = body.spec
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="A profile name is required.")
|
||||
raise refused(400, "profile_name_required")
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
|
||||
DownloadProfile.user_id == user.id
|
||||
@@ -268,16 +265,16 @@ def update_profile(
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="A profile name is required.")
|
||||
raise refused(400, "profile_name_required")
|
||||
p.name = name[:80]
|
||||
if "spec" in sent:
|
||||
if body.spec is None:
|
||||
raise HTTPException(status_code=400, detail="spec must be an object")
|
||||
raise refused(400, "spec_not_object")
|
||||
p.spec = body.spec
|
||||
db.commit()
|
||||
return _serialize_profile(p)
|
||||
@@ -289,7 +286,7 @@ def delete_profile(
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
db.delete(p)
|
||||
db.commit()
|
||||
return {"deleted": profile_id}
|
||||
@@ -319,7 +316,7 @@ def set_credential(
|
||||
"""Store (encrypted) this user's cookies.txt for one supported host. Write-only: the value is
|
||||
never echoed back, logged, or audited — only the fact that it changed."""
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise HTTPException(status_code=404, detail="Unknown host")
|
||||
raise not_found("host")
|
||||
try:
|
||||
cleaned = dlcreds.validate_cookies(body.cookies)
|
||||
except dlcreds.InvalidCookies as e:
|
||||
@@ -341,7 +338,7 @@ def delete_credential(
|
||||
host_key: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise HTTPException(status_code=404, detail="Unknown host")
|
||||
raise not_found("host")
|
||||
if dlcreds.delete_credential(db, user.id, host_key):
|
||||
audit.record(
|
||||
db,
|
||||
@@ -362,7 +359,7 @@ def _resolve_spec(
|
||||
if profile_id is not None:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or (not p.is_builtin and p.user_id != user.id):
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
return p.spec, p.id
|
||||
if spec is not None:
|
||||
return spec, None
|
||||
@@ -373,7 +370,7 @@ def _resolve_spec(
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if fallback is None:
|
||||
raise HTTPException(status_code=400, detail="No download profile available.")
|
||||
raise refused(400, "no_profile")
|
||||
return fallback.spec, fallback.id
|
||||
|
||||
|
||||
@@ -404,7 +401,7 @@ def enqueue_download(
|
||||
status_code=422, detail={"code": "url_blocked", "reason": e.reason}
|
||||
) from e
|
||||
spec, profile_id = _resolve_spec(db, user, body.profile_id, body.spec)
|
||||
display_name = (body.display_name or "").strip() or None
|
||||
display_name = display_text(body.display_name)
|
||||
try:
|
||||
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
|
||||
except quota.QuotaExceeded as e:
|
||||
@@ -412,61 +409,119 @@ def enqueue_download(
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
# Read the upload in 1 MiB chunks so a large file never sits fully in memory, and the size cap is
|
||||
# enforced on the ACTUAL bytes read (a client-set Content-Length can't be trusted to gate it).
|
||||
_UPLOAD_CHUNK = 1024 * 1024
|
||||
# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
|
||||
# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
|
||||
# FastAPI *before the first dependency runs*, so `require_human` could not gate it and an anonymous
|
||||
# caller could make the process write the whole per-file allowance to temp, per request (P1; SEC-20
|
||||
# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
|
||||
# declared AFTER the auth one, is the only shape that closes it — the same one
|
||||
# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
|
||||
# every route with a raised ceiling rather than leaving it to a comment like this one.
|
||||
#
|
||||
# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a
|
||||
# Hungarian filename would otherwise be unsendable.
|
||||
UPLOAD_FILENAME_HEADER = "x-upload-filename"
|
||||
UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name"
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
def upload_download(
|
||||
file: UploadFile = File(...),
|
||||
display_name: str | None = Form(None),
|
||||
@dataclass(frozen=True)
|
||||
class SpooledUpload:
|
||||
"""What the body dependency hands the handler: the bytes already on disk, plus the two names
|
||||
that came alongside them."""
|
||||
|
||||
path: Path
|
||||
size: int
|
||||
user_id: int
|
||||
filename: str
|
||||
display_name: str | None
|
||||
|
||||
|
||||
def _header_text(request: Request, name: str) -> str:
|
||||
"""One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the
|
||||
caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's
|
||||
own fallback names the file."""
|
||||
raw = request.headers.get(name, "")
|
||||
try:
|
||||
return unquote(raw, errors="strict").strip()
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
# silent-ok: a mis-encoded header is the caller's own malformed request, and this one is
|
||||
# cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on
|
||||
# a label; logging it would hand any client a free log line per request.
|
||||
return ""
|
||||
|
||||
|
||||
async def spool_upload(
|
||||
request: Request,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Ingest a local file into the caller's library (E7 S5). Any file type is accepted; media
|
||||
(video/audio) gets the full function set, a generic file is download-only. The file is streamed
|
||||
to a temp path under the download volume (enforcing the per-file cap as it goes), then handed to
|
||||
the non-yt-dlp ingest which probes + places it and returns an already-`done` job.
|
||||
) -> AsyncIterator[SpooledUpload]:
|
||||
"""Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
|
||||
|
||||
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||
plus sync DB writes — so it must NOT run on the event loop. `file.file` is the sync file object
|
||||
Starlette spools the multipart body into."""
|
||||
`user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so
|
||||
naming `require_human` here makes the auth check a precondition of reading the body at all — a
|
||||
stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest
|
||||
MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit,
|
||||
empty, quota, probe or move error, a client that vanished mid-upload — it is removed."""
|
||||
cap = sysconfig.get_int(db, "download_upload_max_bytes")
|
||||
user_id = user.id
|
||||
# Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection`
|
||||
# for why, and for the precondition it enforces. Everything this dependency still needs is
|
||||
# already an int or a header in hand: touching an ORM attribute below would reload `user` and
|
||||
# check a connection straight back out.
|
||||
release_read_connection(db)
|
||||
staging = Path(settings.download_root) / ".staging" / "uploads"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=str(staging))
|
||||
tmp_path = Path(tmp)
|
||||
try:
|
||||
size = 0
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
while chunk := file.file.read(_UPLOAD_CHUNK):
|
||||
size += len(chunk)
|
||||
if cap and size > cap:
|
||||
raise HTTPException(
|
||||
status_code=422, detail={"code": "upload_too_large", "limit": cap}
|
||||
)
|
||||
out.write(chunk)
|
||||
if size == 0:
|
||||
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
||||
try:
|
||||
job = uploadmod.ingest_upload(
|
||||
db,
|
||||
user.id,
|
||||
filename=file.filename or "upload",
|
||||
src_path=tmp_path,
|
||||
size=size,
|
||||
display_name=(display_name or "").strip() or None,
|
||||
size = await capped_stream_to(
|
||||
request,
|
||||
out,
|
||||
cap,
|
||||
HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}),
|
||||
)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||
return _serialize_job(db, job)
|
||||
yield SpooledUpload(
|
||||
path=tmp_path,
|
||||
size=size,
|
||||
user_id=user_id,
|
||||
filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload",
|
||||
display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)),
|
||||
)
|
||||
finally:
|
||||
# Ingest MOVES the temp file into place on success (unlink is then a no-op); on any early exit
|
||||
# — cap hit, empty, quota, probe/move error — this removes the abandoned staging file.
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
def upload_download(
|
||||
upload: SpooledUpload = Depends(spool_upload),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Ingest a local file into the caller's library (E7 S5). Any file type is accepted; media
|
||||
(video/audio) gets the full function set, a generic file is download-only. The body IS the file
|
||||
(see the header note above); `spool_upload` has already authenticated the caller and streamed it
|
||||
to a temp path under the download volume, so what is left here is the non-yt-dlp ingest which
|
||||
probes + places it and returns an already-`done` job.
|
||||
|
||||
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||
plus sync DB writes — so it must NOT run on the event loop. The streaming half is async because
|
||||
reading the body is; keeping them in separate callables is what lets each run where it belongs."""
|
||||
if upload.size == 0:
|
||||
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
||||
try:
|
||||
job = uploadmod.ingest_upload(
|
||||
db,
|
||||
upload.user_id,
|
||||
filename=upload.filename,
|
||||
src_path=upload.path,
|
||||
size=upload.size,
|
||||
display_name=upload.display_name,
|
||||
)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
class EnqueueEditIn(BaseModel):
|
||||
source_job_id: int = 0
|
||||
edit_spec: dict
|
||||
@@ -484,9 +539,9 @@ def enqueue_edit(
|
||||
source file is only read, never modified, so editing a shared video is safe."""
|
||||
source_job = _accessible_job(db, user, body.source_job_id)
|
||||
if source_job.status != "done" or source_job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="You can only edit a finished download.")
|
||||
raise refused(409, "edit_unfinished")
|
||||
spec = body.edit_spec
|
||||
display_name = (body.display_name or "").strip() or None
|
||||
display_name = display_text(body.display_name)
|
||||
try:
|
||||
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
|
||||
except quota.QuotaExceeded as e:
|
||||
@@ -522,7 +577,20 @@ def list_downloads(
|
||||
|
||||
@router.get("/usage")
|
||||
def my_usage(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
return quota.usage(db, user.id)
|
||||
"""The caller's download quota, plus the per-file upload cap.
|
||||
|
||||
The cap rides here rather than in `quota.usage` because it is not a quota value — it is a
|
||||
`sysconfig` knob about one route — and this is the route layer, where composing the two is the
|
||||
job. The upload dialog needs it to refuse an over-cap file BEFORE sending it: both server-side
|
||||
caps answer too late to be pleasant (the edge ceiling refuses the request outright, see
|
||||
`bodylimit.ROUTE_MAX_BODY_BYTES`), and making the user push 600 MB up the wire to be told the
|
||||
limit is 500 is a bad answer even when it is a correct one. It is a convenience, not the
|
||||
boundary: the caps stay where they are and still refuse a client that ignores this.
|
||||
"""
|
||||
return {
|
||||
**quota.usage(db, user.id),
|
||||
"upload_max_bytes": sysconfig.get_int(db, "download_upload_max_bytes"),
|
||||
}
|
||||
|
||||
|
||||
# Priority when a source has several jobs (e.g. a done one plus a fresh queued one).
|
||||
@@ -614,12 +682,12 @@ def update_download_meta(
|
||||
) -> dict:
|
||||
"""Update a download's display metadata: title, channel name + link, and extra reference URLs.
|
||||
All are display-only overrides — the on-disk file keeps its system-decided, id-bound name."""
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
sent = body.model_fields_set
|
||||
if "display_name" in sent:
|
||||
job.display_name = ((body.display_name or "").strip())[:255] or None
|
||||
job.display_name = display_text(body.display_name)
|
||||
if "display_uploader" in sent:
|
||||
job.display_uploader = ((body.display_uploader or "").strip())[:255] or None
|
||||
job.display_uploader = display_text(body.display_uploader)
|
||||
if "display_uploader_url" in sent:
|
||||
job.display_uploader_url = _clean_url(body.display_uploader_url)
|
||||
if "extra_links" in sent:
|
||||
@@ -668,9 +736,9 @@ def redownload_download(
|
||||
if job.job_kind in ("edit", "upload"):
|
||||
# Neither has a re-fetchable source: an editor clip is derived locally, an upload IS the
|
||||
# source. Re-running yt-dlp against source_kind='upload'/'edit' would just fail.
|
||||
raise HTTPException(status_code=409, detail="This item can't be re-downloaded.")
|
||||
raise refused(409, "not_redownloadable")
|
||||
if job.status in ("queued", "running", "paused"):
|
||||
raise HTTPException(status_code=409, detail="This download is still in progress.")
|
||||
raise refused(409, "still_downloading")
|
||||
service.redownload(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -699,7 +767,7 @@ def delete_download(
|
||||
def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
raise not_found("download")
|
||||
if job.user_id == user.id:
|
||||
return job
|
||||
shared = db.execute(
|
||||
@@ -708,7 +776,7 @@ def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
)
|
||||
).first()
|
||||
if shared is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
@@ -722,10 +790,10 @@ def download_file(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=409, detail="This download isn't ready.")
|
||||
raise refused(409, "download_not_ready")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="File is no longer available.")
|
||||
raise not_found("file")
|
||||
|
||||
filename = storage.download_filename(
|
||||
job.display_name or asset.title or asset.source_ref, asset.container, path
|
||||
@@ -764,7 +832,7 @@ def storyboard_meta(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=409, detail="This download isn't ready.")
|
||||
raise refused(409, "download_not_ready")
|
||||
meta = editmod.ensure_storyboard(asset, settings.download_root)
|
||||
if meta is None:
|
||||
return {"available": False}
|
||||
@@ -785,10 +853,10 @@ def poster_image(job_id: int, user: User = Depends(require_human), db: Session =
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.poster_path:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
|
||||
@@ -799,10 +867,10 @@ def storyboard_image(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.storyboard_path:
|
||||
raise HTTPException(status_code=404, detail="No filmstrip.")
|
||||
raise not_found("filmstrip")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.storyboard_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No filmstrip.")
|
||||
raise not_found("filmstrip")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
|
||||
@@ -820,12 +888,12 @@ def share_download(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="Only a finished download can be shared.")
|
||||
raise refused(409, "share_unfinished")
|
||||
target = body.email.strip().lower()
|
||||
if not target:
|
||||
raise HTTPException(status_code=400, detail="A recipient email is required.")
|
||||
raise refused(400, "recipient_required")
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == target)
|
||||
).scalar_one_or_none()
|
||||
@@ -834,7 +902,7 @@ def share_download(
|
||||
if not is_messageable_user(recipient):
|
||||
raise not_found("recipient")
|
||||
if recipient.id == user.id:
|
||||
raise HTTPException(status_code=400, detail="That's already your download.")
|
||||
raise refused(400, "share_own")
|
||||
exists = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == recipient.id
|
||||
@@ -853,7 +921,7 @@ def share_download(
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
return {"shared_with": recipient.email}
|
||||
return {"ok": True, "shared_with": recipient.email}
|
||||
|
||||
|
||||
@router.delete("/{job_id}/share/{email}")
|
||||
@@ -863,7 +931,9 @@ def unshare_download(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
_own_job(db, user, job_id) # ownership guard (404s if not the caller's job)
|
||||
owned_or_404(
|
||||
db, DownloadJob, job_id, user, what="download"
|
||||
) # ownership guard (404s if not the caller's job)
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == email.strip().lower())
|
||||
).scalar_one_or_none()
|
||||
@@ -895,7 +965,7 @@ def remove_shared_with_me(
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"removed": job_id}
|
||||
return {"ok": True, "removed": job_id}
|
||||
|
||||
|
||||
# --- recipients (for the internal "share with a user" picker) ------------------------------
|
||||
@@ -925,10 +995,10 @@ def share_recipients(
|
||||
def _own_link(db: Session, user: User, link_id: int) -> DownloadLink:
|
||||
link = db.get(DownloadLink, link_id)
|
||||
if link is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown link")
|
||||
raise not_found("link")
|
||||
job = db.get(DownloadJob, link.job_id)
|
||||
if job is None or job.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown link")
|
||||
raise not_found("link")
|
||||
return link
|
||||
|
||||
|
||||
@@ -944,7 +1014,7 @@ def _link_expiry(days: int | None) -> datetime | None:
|
||||
def list_links(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
_own_job(db, user, job_id)
|
||||
owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
rows = (
|
||||
db.execute(
|
||||
select(DownloadLink)
|
||||
@@ -975,9 +1045,9 @@ def create_link(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="Only a finished download can be shared.")
|
||||
raise refused(409, "share_unfinished")
|
||||
pw = body.password.strip()
|
||||
link = DownloadLink(
|
||||
job_id=job_id,
|
||||
|
||||
+13
-12
@@ -1,6 +1,6 @@
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
@@ -29,10 +29,11 @@ from app.models import (
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.routes._common import get_or_404
|
||||
from app.routes._common import get_or_404, youtube_refusal, not_found
|
||||
from app.sync.videos import parse_iso8601_duration
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["feed"])
|
||||
|
||||
@@ -120,7 +121,7 @@ def get_feed(
|
||||
vals, cursor_id = decode_cursor(keys, cursor)
|
||||
except ValueError:
|
||||
# `from None`: the ValueError is an expected bad-input signal, not an error to chain.
|
||||
raise HTTPException(status_code=400, detail="Invalid feed cursor") from None
|
||||
raise refused(400, "invalid_cursor") from None
|
||||
for k, v in zip(keys, vals):
|
||||
k["value"] = v
|
||||
query = query.where(keyset_predicate(keys, cursor_id))
|
||||
@@ -235,7 +236,7 @@ def set_video_state(
|
||||
) -> dict:
|
||||
status = body.status
|
||||
if status not in VALID_STATES:
|
||||
raise HTTPException(status_code=400, detail=f"status must be one of {VALID_STATES}")
|
||||
raise refused(400, "invalid_status")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
@@ -260,7 +261,7 @@ def set_video_state(
|
||||
# or moved this row. The state the caller asked for is the state that holds either way.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "status": "new"}
|
||||
return {"ok": True, "video_id": video_id, "status": "new"}
|
||||
|
||||
# watched / hidden: atomic upsert. The player marks "watched" at end-of-playback at
|
||||
# the same instant a progress checkpoint may DELETE the in-progress row; a plain
|
||||
@@ -281,7 +282,7 @@ def set_video_state(
|
||||
)
|
||||
db.execute(stmt)
|
||||
db.commit()
|
||||
return {"video_id": video_id, "status": status}
|
||||
return {"ok": True, "video_id": video_id, "status": status}
|
||||
|
||||
|
||||
@router.delete("/videos/{video_id}/state")
|
||||
@@ -303,7 +304,7 @@ def clear_video_state(
|
||||
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
|
||||
|
||||
class VideoProgressIn(BaseModel):
|
||||
@@ -361,7 +362,7 @@ def set_video_progress(
|
||||
# silent-ok: exactly the race the comment above describes; the response stays true.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "position_seconds": 0}
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": 0}
|
||||
|
||||
if row is None:
|
||||
row = VideoState(user_id=user.id, video_id=video_id)
|
||||
@@ -369,7 +370,7 @@ def set_video_progress(
|
||||
row.position_seconds = position
|
||||
row.progress_updated_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
return {"video_id": video_id, "position_seconds": position}
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": position}
|
||||
|
||||
|
||||
@router.get("/videos/{video_id}")
|
||||
@@ -402,7 +403,7 @@ def get_video_detail(
|
||||
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit, so the
|
||||
# demo account can't reach it — it just sees "unknown" for a video we don't already store.
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise not_found("video")
|
||||
|
||||
try:
|
||||
with (
|
||||
@@ -411,9 +412,9 @@ def get_video_detail(
|
||||
):
|
||||
items = yt.get_videos([video_id])
|
||||
except YouTubeError as exc:
|
||||
raise HTTPException(status_code=422, detail=f"YouTube lookup failed: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
if not items:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise not_found("video")
|
||||
|
||||
snippet = items[0].get("snippet", {})
|
||||
stats = items[0].get("statistics", {})
|
||||
|
||||
+21
-14
@@ -17,13 +17,16 @@ from app.auth import (
|
||||
optional_current_user,
|
||||
purge_user,
|
||||
)
|
||||
from app import diagnostics
|
||||
from app import debugsession, diagnostics
|
||||
from app.issues import gitea
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import sysconfig
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/me", tags=["me"])
|
||||
|
||||
@@ -70,7 +73,7 @@ def switch_account(
|
||||
here) are allowed, and access is re-checked in case the invite was revoked since."""
|
||||
target = body.user_id
|
||||
if target not in (request.session.get("account_ids") or []):
|
||||
raise HTTPException(status_code=403, detail="Not an account you've signed into here.")
|
||||
raise refused(403, "not_your_account")
|
||||
# An account `_session_user` would refuse — gone, deactivated, suspended, or its session revoked — is
|
||||
# refused here too, and dropped from the switch list the way `_session_user` drops it. Made the session
|
||||
# default instead, the next request's current_user would refuse it as the DEFAULT and clear the whole
|
||||
@@ -78,22 +81,20 @@ def switch_account(
|
||||
u = db.get(User, target)
|
||||
if u is None:
|
||||
drop_from_wallet(request.session, target)
|
||||
raise HTTPException(status_code=404, detail="That account no longer exists.")
|
||||
raise not_found("account")
|
||||
if not is_allowed(db, u.email):
|
||||
raise HTTPException(status_code=403, detail="That account no longer has access.")
|
||||
raise refused(403, "account_no_access")
|
||||
if not u.is_active:
|
||||
drop_from_wallet(request.session, target)
|
||||
raise HTTPException(status_code=403, detail="That account no longer has access.")
|
||||
raise refused(403, "account_no_access")
|
||||
if u.is_suspended:
|
||||
drop_from_wallet(request.session, target)
|
||||
raise HTTPException(status_code=403, detail="That account is suspended.")
|
||||
raise refused(403, "account_suspended")
|
||||
# Revoked: a password reset/change or "log out everywhere" bumped the account's epoch.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(target), 0)) != (u.session_epoch or 0):
|
||||
drop_from_wallet(request.session, target)
|
||||
raise HTTPException(
|
||||
status_code=403, detail="That account was signed out elsewhere — sign in again."
|
||||
)
|
||||
raise refused(403, "account_signed_out")
|
||||
request.session["user_id"] = target
|
||||
return {"ok": True, "user_id": target}
|
||||
|
||||
@@ -158,9 +159,18 @@ def get_me(
|
||||
# The consented diagnostics level in force and when it lapses (E22 S5). The client applies
|
||||
# it to its own log and streams that log while it is raised.
|
||||
"diagnostics": diagnostics.state(db, user),
|
||||
# The live debug session, if one is recording (E16) — a reload resumes it.
|
||||
"debug_session": _live_session(db, user),
|
||||
# Where a bug report goes (E16 S4): the tracker's repository, or None — download only.
|
||||
"issues": gitea.target(db),
|
||||
}
|
||||
|
||||
|
||||
def _live_session(db: Session, user: User) -> dict | None:
|
||||
session = debugsession.open_session(db, user)
|
||||
return None if session is None else debugsession.view(db, session)
|
||||
|
||||
|
||||
@router.delete("/account")
|
||||
def delete_my_account(
|
||||
request: Request,
|
||||
@@ -181,10 +191,7 @@ def delete_my_account(
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "delete_account"})
|
||||
if user.role == "admin" and count_admins(db) <= 1:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="You're the only admin — promote another admin before deleting your account.",
|
||||
)
|
||||
raise refused(400, "only_admin_delete")
|
||||
user_id = user.id
|
||||
# Full erasure (cascades) + access-request cleanup + Google-grant revocation; shared with the
|
||||
# admin delete path. Capture the id first — `user` is detached once purge_user deletes the row.
|
||||
@@ -217,7 +224,7 @@ def update_preferences(
|
||||
user.preferences = merged
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return {"preferences": merged}
|
||||
return {"ok": True, "preferences": merged}
|
||||
|
||||
|
||||
class DiagnosticsIn(BaseModel):
|
||||
|
||||
@@ -16,7 +16,7 @@ routing and unread counts.
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect
|
||||
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, case, delete, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -29,6 +29,8 @@ from app.realtime import manager
|
||||
from app.security import verify_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/messages", tags=["messages"])
|
||||
|
||||
@@ -152,7 +154,7 @@ def setup_keys(
|
||||
"""Store this user's public key + passphrase-wrapped private key. Set-once: changing the
|
||||
key would orphan every existing conversation key, so re-setup is rejected here."""
|
||||
if db.get(MessageKey, user.id) is not None:
|
||||
raise HTTPException(status_code=409, detail="Secure messaging is already set up.")
|
||||
raise refused(409, "messaging_set_up")
|
||||
db.add(
|
||||
MessageKey(
|
||||
user_id=user.id,
|
||||
@@ -164,7 +166,7 @@ def setup_keys(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"configured": True}
|
||||
return {"ok": True, "configured": True}
|
||||
|
||||
|
||||
@router.post("/keys/reset")
|
||||
@@ -185,11 +187,11 @@ def reset_keys(
|
||||
less (Google-only) account has nothing to re-verify, so it relies on its live session plus the
|
||||
client-side danger confirmation."""
|
||||
if not _reset_key_limiter.allow(str(user.id)):
|
||||
raise HTTPException(status_code=429, detail="Too many attempts. Try again shortly.")
|
||||
raise refused(429, "too_many_attempts")
|
||||
if user.password_hash and not verify_password(
|
||||
payload.current_password or "", user.password_hash
|
||||
):
|
||||
raise HTTPException(status_code=403, detail="Current password is incorrect.")
|
||||
raise refused(403, "wrong_password")
|
||||
# Delete the now-dead ciphertext first (both directions), then the wrapped keypair itself so a
|
||||
# fresh setup is allowed. Order doesn't matter (no FK between them), but do it in one commit.
|
||||
db.execute(
|
||||
@@ -202,7 +204,7 @@ def reset_keys(
|
||||
if key is not None:
|
||||
db.delete(key)
|
||||
db.commit()
|
||||
return {"configured": False}
|
||||
return {"ok": True, "configured": False}
|
||||
|
||||
|
||||
@router.get("/keys/{user_id}")
|
||||
@@ -223,10 +225,10 @@ def public_key(
|
||||
p = db.get(User, user_id)
|
||||
has_history = db.scalar(select(Message.id).where(cond).limit(1))
|
||||
if p is None or (not is_messageable_user(p) and not has_history):
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
raise not_found("user")
|
||||
k = db.get(MessageKey, user_id)
|
||||
if k is None:
|
||||
raise HTTPException(status_code=404, detail="That member hasn't set up messaging yet.")
|
||||
raise refused(404, "no_messaging")
|
||||
return {"user_id": user_id, "public_key": k.public_key}
|
||||
|
||||
|
||||
@@ -344,7 +346,7 @@ def get_thread(
|
||||
# still opens). Otherwise the same 404 as a non-existent id → no enumeration oracle.
|
||||
has_history = db.scalar(select(Message.id).where(cond).limit(1))
|
||||
if p is None or (not is_messageable_user(p) and not has_history):
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
raise not_found("user")
|
||||
partner = _serialize_user(p)
|
||||
msgs = (
|
||||
db.execute(select(Message).where(cond).order_by(Message.created_at.asc())).scalars().all()
|
||||
@@ -374,20 +376,20 @@ def send_message(
|
||||
"""Send an end-to-end-encrypted message. The server stores only the ciphertext the client
|
||||
produced, then pushes it live to the recipient (and the sender's other devices)."""
|
||||
if not _send_limiter.allow(f"msg:{user.id}"):
|
||||
raise HTTPException(status_code=429, detail="Too many messages — slow down a moment.")
|
||||
raise refused(429, "message_rate")
|
||||
ct = (payload.ciphertext or "").strip()
|
||||
iv = (payload.iv or "").strip()
|
||||
if not ct or not iv:
|
||||
raise HTTPException(status_code=400, detail="Message is empty.")
|
||||
raise refused(400, "message_empty")
|
||||
if len(ct) > MAX_CIPHERTEXT:
|
||||
raise HTTPException(status_code=400, detail="Message is too long.")
|
||||
raise refused(400, "message_too_long")
|
||||
if payload.recipient_id == user.id:
|
||||
raise HTTPException(status_code=400, detail="You can't message yourself.")
|
||||
raise refused(400, "message_self")
|
||||
recipient = db.get(User, payload.recipient_id)
|
||||
if not is_messageable_user(recipient):
|
||||
raise HTTPException(status_code=404, detail="Recipient not available.")
|
||||
raise not_found("recipient")
|
||||
if db.get(MessageKey, recipient.id) is None:
|
||||
raise HTTPException(status_code=404, detail="That member hasn't set up messaging yet.")
|
||||
raise refused(404, "no_messaging")
|
||||
m = Message(kind="user", sender_id=user.id, recipient_id=recipient.id, ciphertext=ct, iv=iv)
|
||||
db.add(m)
|
||||
db.commit()
|
||||
|
||||
@@ -5,7 +5,7 @@ All endpoints use `current_user` (not `require_human`): notifications and their
|
||||
state are personal UI state with no quota or YouTube involvement, so the shared demo account
|
||||
manages its own inbox normally."""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -14,6 +14,7 @@ from app.db import get_db
|
||||
from app.models import Notification, User
|
||||
from app.notifications import trim_read
|
||||
from app.utils import iso
|
||||
from app.routes._common import owned_or_404
|
||||
|
||||
router = APIRouter(prefix="/api/me/notifications", tags=["notifications"])
|
||||
|
||||
@@ -72,20 +73,13 @@ def unread_count(user: User = Depends(current_user), db: Session = Depends(get_d
|
||||
return {"count": count or 0}
|
||||
|
||||
|
||||
def _owned(db: Session, notification_id: int, user: User) -> Notification:
|
||||
n = db.get(Notification, notification_id)
|
||||
if n is None or n.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Notification not found")
|
||||
return n
|
||||
|
||||
|
||||
@router.post("/{notification_id}/read")
|
||||
def mark_read(
|
||||
notification_id: int,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
n = _owned(db, notification_id, user)
|
||||
n = owned_or_404(db, Notification, notification_id, user, what="notification")
|
||||
n.read = True
|
||||
db.commit()
|
||||
trim_read(db, user.id)
|
||||
@@ -110,7 +104,7 @@ def dismiss(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
n = _owned(db, notification_id, user)
|
||||
n = owned_or_404(db, Notification, notification_id, user, what="notification")
|
||||
# Dismiss implies read (it no longer counts toward the unread badge).
|
||||
n.dismissed = True
|
||||
n.read = True
|
||||
|
||||
+162
-85
@@ -3,10 +3,8 @@ CRUD (create / rename / delete, add / remove / reorder items) plus YouTube two-w
|
||||
mirror the user's own YouTube playlists in (`/sync-youtube`) and push/revert local edits back
|
||||
out (`/{id}/push-plan`, `/{id}/revert-youtube`), behind the optional write scope."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, select, and_
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
@@ -16,14 +14,18 @@ from app.db import get_db
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
|
||||
from app.sync.playlists import (
|
||||
PlaylistGone,
|
||||
lock_playlists,
|
||||
merge_playlist,
|
||||
plan_merge,
|
||||
recompute_dirty,
|
||||
pull_user_playlists,
|
||||
repull_playlist,
|
||||
sync_user_playlists,
|
||||
sync_claim,
|
||||
user_sync_claim,
|
||||
)
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import get_or_404, owned_or_404
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
|
||||
router = APIRouter(prefix="/api/playlists", tags=["playlists"])
|
||||
@@ -48,10 +50,6 @@ def _check_pull_cooldown(user: User) -> None:
|
||||
raise HTTPException(status_code=429, detail={"code": "sync_cooldown"})
|
||||
|
||||
|
||||
def _own_playlist(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
return owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
|
||||
|
||||
def _refused(status: int, reason: str) -> HTTPException:
|
||||
"""A playlist action the server refuses, as a reason the UI translates
|
||||
(`playlists.errors.<reason>`) — a bare English detail reached the Hungarian UI verbatim."""
|
||||
@@ -64,8 +62,11 @@ def _touch(pl: Playlist) -> None:
|
||||
`UpdatedAtMixin`'s `onupdate` fires on an UPDATE of the PLAYLIST row, and add / remove /
|
||||
reorder / move write the item rows instead — so without this the rail's "recently changed"
|
||||
sort would only ever see renames. The assignment is what makes the column mean "last time
|
||||
this list changed", which is what the rail promises the reader."""
|
||||
pl.updated_at = datetime.now(timezone.utc)
|
||||
this list changed", which is what the rail promises the reader.
|
||||
|
||||
The DATABASE's clock, like the mixin's own `onupdate`: one sort key written from two clocks
|
||||
(this process's and Postgres's) could order a rename and an item edit the wrong way round."""
|
||||
pl.updated_at = func.now()
|
||||
|
||||
|
||||
def _next_playlist_position(db: Session, user_id: int) -> int:
|
||||
@@ -81,7 +82,10 @@ def _next_playlist_position(db: Session, user_id: int) -> int:
|
||||
|
||||
|
||||
def _next_item_position(db: Session, playlist_id: int) -> int:
|
||||
"""Append position for a new item at the end of a playlist."""
|
||||
"""Append position for a new item at the end of a playlist. Locks the playlist row first: two
|
||||
adds (two tabs, the rail and the page) otherwise both read the same max and land on one
|
||||
position, and the list's order between them is then arbitrary."""
|
||||
lock_playlists(db, playlist_id)
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
|
||||
@@ -94,7 +98,10 @@ def _next_item_position(db: Session, playlist_id: int) -> int:
|
||||
|
||||
def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Append a video to a playlist if not already present (idempotent). Returns whether it
|
||||
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
||||
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later).
|
||||
Locked BEFORE the presence check: two adds of one video both saw "absent" and the second one's
|
||||
insert hit uq_playlist_video as a 500."""
|
||||
lock_playlists(db, pl.id)
|
||||
exists = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||
@@ -108,13 +115,14 @@ def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) ->
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
return True
|
||||
|
||||
|
||||
def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Remove a video from a playlist if present (idempotent). Returns whether it was removed.
|
||||
`mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
||||
lock_playlists(db, pl.id)
|
||||
item = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||
@@ -126,7 +134,7 @@ def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool)
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
return True
|
||||
|
||||
|
||||
@@ -280,7 +288,7 @@ def _watch_later(db: Session, user: User) -> Playlist:
|
||||
position=_next_playlist_position(db, user.id),
|
||||
)
|
||||
db.add(pl)
|
||||
db.commit()
|
||||
db.flush() # the id, for the item the caller adds next; the handler commits
|
||||
return pl
|
||||
|
||||
|
||||
@@ -295,7 +303,8 @@ def add_watch_later(
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
pl = _watch_later(db, user)
|
||||
_add_item(db, pl, video_id, mark_dirty=False)
|
||||
return {"saved": True, "playlist_id": pl.id}
|
||||
db.commit()
|
||||
return {"ok": True, "saved": True, "playlist_id": pl.id}
|
||||
|
||||
|
||||
@router.delete("/watch-later/{video_id}")
|
||||
@@ -309,7 +318,8 @@ def remove_watch_later(
|
||||
)
|
||||
if pl is not None:
|
||||
_remove_item(db, pl, video_id, mark_dirty=False)
|
||||
return {"saved": False}
|
||||
db.commit()
|
||||
return {"ok": True, "saved": False}
|
||||
|
||||
|
||||
@router.post("/sync-youtube")
|
||||
@@ -321,9 +331,18 @@ def sync_youtube(
|
||||
Also runs automatically on the scheduler; this is the manual 'sync now'."""
|
||||
if not has_read_scope(user):
|
||||
raise _refused(403, "youtube_read_scope")
|
||||
_check_pull_cooldown(user)
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return sync_user_playlists(db, user)
|
||||
with user_sync_claim(db, user.id) as free:
|
||||
if not free:
|
||||
# Another run (the scheduler, another tab) is pulling right now: "0 synced" would lie.
|
||||
raise _refused(409, "sync_all_busy")
|
||||
# After the claim, like the re-pull's: a refused attempt must not spend one of the pulls.
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return {"ok": True, **pull_user_playlists(db, user)}
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
@router.post("/{playlist_id}/revert-youtube")
|
||||
@@ -335,24 +354,31 @@ def revert_youtube(
|
||||
"""Reset one YouTube-linked playlist to its current YouTube state, discarding local
|
||||
edits (and clearing the dirty flag). The user-facing 'undo all my changes' for a list
|
||||
once the in-session undo history is gone."""
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if not pl.yt_playlist_id:
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_read_scope(user):
|
||||
raise _refused(403, "youtube_read_scope")
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return repull_playlist(db, user, pl)
|
||||
except YouTubeError:
|
||||
db.rollback()
|
||||
raise _refused(422, "youtube_failed")
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
raise _refused(409, "sync_busy")
|
||||
# The cooldown is spent only by a pull that runs: checked after the claim, so clicking
|
||||
# Revert while a push is in flight does not use up the three pulls per ten minutes.
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return {"ok": True, **repull_playlist(db, user, pl)}
|
||||
except PlaylistGone:
|
||||
raise not_found("playlist")
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
"""A local playlist the user may push to YouTube (create or update). The built-in
|
||||
Watch later and read-only YouTube mirrors are never pushable."""
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if pl.kind == "watch_later":
|
||||
raise _refused(400, "watch_later_not_pushable")
|
||||
return pl
|
||||
@@ -371,8 +397,8 @@ def push_plan(
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
plan = plan_merge(db, user, pl)
|
||||
except YouTubeError:
|
||||
raise _refused(422, "youtube_failed")
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc)
|
||||
plan["remaining_today"] = quota.remaining_today(db)
|
||||
plan["affordable"] = plan["units_estimate"] <= plan["remaining_today"]
|
||||
return plan
|
||||
@@ -388,25 +414,32 @@ def push(
|
||||
neither side loses an item). Refuses if the estimated quota exceeds what's left today, so a
|
||||
big sync can't strand half-applied. Write scope required (require_write_scope)."""
|
||||
pl = _pushable(db, user, playlist_id)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
||||
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||
live = None
|
||||
if pl.yt_playlist_id:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
live = (
|
||||
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||
)
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise _refused(429, "quota_exceeded")
|
||||
return merge_playlist(db, user, pl, live=live)
|
||||
except YouTubeError:
|
||||
db.rollback()
|
||||
raise _refused(422, "youtube_failed")
|
||||
# One sync of a list at a time, across tabs, devices and the scheduler (`sync_claim`): the
|
||||
# client's own in-flight flag only covers the menus of one page.
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
raise _refused(409, "sync_busy")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
||||
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||
live = None
|
||||
if pl.yt_playlist_id:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
live = (
|
||||
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||
)
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise _refused(429, "quota_exceeded")
|
||||
return {"ok": True, **merge_playlist(db, user, pl, live=live)}
|
||||
except PlaylistGone:
|
||||
raise not_found("playlist")
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
@router.get("/{playlist_id}")
|
||||
@@ -415,7 +448,7 @@ def get_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
state = aliased(VideoState)
|
||||
rows = db.execute(
|
||||
select(*feed_columns(user, state))
|
||||
@@ -423,7 +456,8 @@ def get_playlist(
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id))
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
.order_by(PlaylistItem.position)
|
||||
# `id` breaks a position tie the same way on every read (the renumbering add does too).
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).all()
|
||||
return {
|
||||
"id": pl.id,
|
||||
@@ -447,7 +481,8 @@ def rename_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
# Locked: the new name moves `dirty`, which the scheduled mirror refresh reads under this lock.
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||
if "name" in body.model_fields_set:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
@@ -484,37 +519,70 @@ def delete_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if pl.kind == "watch_later":
|
||||
raise _refused(400, "watch_later_undeletable")
|
||||
if on_youtube:
|
||||
if not pl.yt_playlist_id:
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_write_scope(user):
|
||||
raise _refused(403, "youtube_editing_off")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt.delete_playlist(pl.yt_playlist_id)
|
||||
except YouTubeError:
|
||||
raise _refused(422, "youtube_failed")
|
||||
db.delete(pl) # items cascade
|
||||
db.commit()
|
||||
# Not in the middle of a sync of it (`sync_claim`): a push that read the list before the delete
|
||||
# and wrote YouTube after it planned from no items at all, so it emptied the YouTube playlist a
|
||||
# local-only delete means to keep. The push checks for a vanished list too (`PlaylistGone`);
|
||||
# this refuses the delete while a sync runs rather than race it.
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
raise _refused(409, "sync_busy")
|
||||
if on_youtube:
|
||||
if not pl.yt_playlist_id:
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_write_scope(user):
|
||||
raise _refused(403, "youtube_editing_off")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt.delete_playlist(pl.yt_playlist_id)
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc)
|
||||
# The playlist first, as every item write: the cascade deletes the items before the list row.
|
||||
lock_playlists(db, pl.id)
|
||||
db.delete(pl) # items cascade
|
||||
db.commit()
|
||||
return {"deleted": playlist_id}
|
||||
|
||||
|
||||
class AddItemIn(VideoRefIn):
|
||||
# Where the video lands among the list's items, in their stored order (0 = first). Omitted, it
|
||||
# is appended. The list page's "Undo" of a removal puts the row back where it was with this —
|
||||
# ONE request, because append-then-reorder leaves it at the end whenever the second call fails.
|
||||
position: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
@router.post("/{playlist_id}/items")
|
||||
def add_item(
|
||||
playlist_id: int,
|
||||
body: VideoRefIn,
|
||||
body: AddItemIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
video_id = body.video_id
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
_add_item(db, pl, video_id, mark_dirty=True)
|
||||
return {"playlist_id": pl.id, "video_id": video_id}
|
||||
added = _add_item(db, pl, video_id, mark_dirty=False)
|
||||
if added and body.position is not None:
|
||||
# Renumber the whole list with the new row at `position` (clamped to the end): positions
|
||||
# can have gaps after removals, so "index N" is only meaningful over the ordered rows.
|
||||
rows = list(
|
||||
db.execute(
|
||||
select(PlaylistItem)
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).scalars()
|
||||
)
|
||||
new = next(r for r in rows if r.video_id == video_id)
|
||||
rows.remove(new)
|
||||
rows.insert(min(body.position, len(rows)), new)
|
||||
for pos, row in enumerate(rows):
|
||||
row.position = pos
|
||||
recompute_dirty(db, pl)
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
|
||||
@router.delete("/{playlist_id}/items/{video_id}")
|
||||
@@ -524,9 +592,10 @@ def remove_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
_remove_item(db, pl, video_id, mark_dirty=True)
|
||||
return {"playlist_id": pl.id, "video_id": video_id}
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
|
||||
class MoveItemIn(BaseModel):
|
||||
@@ -548,7 +617,7 @@ def move_item(
|
||||
exactly the kind of half-applied state this codebase treats as a bug rather than an edge
|
||||
case. Idempotent: if the video is already in the target, the source row is simply dropped
|
||||
(moving into a list that already has it is a removal, not a duplicate)."""
|
||||
src = _own_playlist(db, user, playlist_id)
|
||||
src = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
# Its own reason: "this playlist doesn't exist any more" about a TARGET that is gone was read as
|
||||
# the list on screen — which is still there.
|
||||
dst = owned_or_404(db, Playlist, body.target_playlist_id, user, what="move_target")
|
||||
@@ -556,6 +625,8 @@ def move_item(
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "playlist_move", "reason": "same_playlist"}
|
||||
)
|
||||
# Both lists, in id order: a move A→B beside a move B→A would otherwise lock crosswise.
|
||||
lock_playlists(db, src.id, dst.id)
|
||||
item = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == src.id, PlaylistItem.video_id == video_id
|
||||
@@ -571,19 +642,24 @@ def move_item(
|
||||
PlaylistItem.playlist_id == dst.id, PlaylistItem.video_id == video_id
|
||||
)
|
||||
)
|
||||
if already is not None:
|
||||
db.delete(item) # the uq_playlist_video constraint would reject a second row anyway
|
||||
else:
|
||||
dst_changed = False
|
||||
if already is None:
|
||||
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
|
||||
# survives the move, and there is no window where the video belongs to neither list.
|
||||
# survives the move, and there is no window where the video belongs to neither list. The
|
||||
# check above holds: both lists are locked, and an insert into the target (another tab's
|
||||
# add, a YouTube merge) waits for the lock — its foreign key needs the target row.
|
||||
item.playlist_id = dst.id
|
||||
item.position = _next_item_position(db, dst.id)
|
||||
# Both sides changed, so both get the dirty recompute and the timestamp — the asymmetric
|
||||
# version of this (touching only the source) is the recurring smell CLAUDE.md warns about.
|
||||
dst_changed = True
|
||||
if not dst_changed:
|
||||
db.delete(item)
|
||||
# The source always changed; the target only when the row landed in it. `_touch` on a target
|
||||
# that already had the video promoted an untouched list in the rail's "recently changed" sort.
|
||||
recompute_dirty(db, src)
|
||||
recompute_dirty(db, dst)
|
||||
_touch(src)
|
||||
_touch(dst)
|
||||
if dst_changed:
|
||||
recompute_dirty(db, dst)
|
||||
_touch(dst)
|
||||
db.commit()
|
||||
return {"ok": True, "from_playlist_id": src.id, "to_playlist_id": dst.id, "video_id": video_id}
|
||||
|
||||
@@ -599,7 +675,8 @@ def reorder_items(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
# Locked: a concurrent add between reading the items and writing the order would collide.
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||
order = body.video_ids
|
||||
items = {
|
||||
it.video_id: it
|
||||
@@ -625,4 +702,4 @@ def reorder_items(
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.commit()
|
||||
return {"playlist_id": pl.id, "count": pos}
|
||||
return {"ok": True, "playlist_id": pl.id, "count": pos}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
"""Shared helpers, serializers and constants for the plex route package."""
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import PlexItem
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.routes._common import not_found
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
@@ -13,7 +13,7 @@ log = logs.get_logger(__name__)
|
||||
def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
||||
it = by_rating_key(db, PlexItem, rating_key)
|
||||
if it is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex item")
|
||||
raise not_found("plex_item")
|
||||
return it
|
||||
|
||||
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -20,6 +20,9 @@ from app.plex import sync as plex_sync
|
||||
from app.plex import watch_sync as plex_watch
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
@@ -35,10 +38,15 @@ def test_connection(_: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
info = plex.server_info()
|
||||
sections = plex.sections()
|
||||
media_check = _media_check(db, plex, sections)
|
||||
except PlexNotConfigured as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except PlexNotConfigured:
|
||||
raise refused(400, "plex_not_configured") from None
|
||||
except PlexError as e:
|
||||
raise HTTPException(status_code=422, detail=f"Plex connection failed: {e}")
|
||||
# What the upstream said is for the log, not the toast: it can quote the server URL back.
|
||||
log.warning(
|
||||
"the Plex connection test failed",
|
||||
extra=logs.event("plex.admin.connection_failed", reason=type(e).__name__, error=str(e)),
|
||||
)
|
||||
raise refused(422, "plex_unreachable") from None
|
||||
return {
|
||||
"ok": True,
|
||||
"server_name": info.get("friendlyName") or info.get("title1") or "Plex",
|
||||
@@ -95,7 +103,7 @@ def _media_check(db: Session, plex: PlexClient, sections: list[dict]) -> dict:
|
||||
def trigger_sync(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Admin: mirror the enabled Plex sections into the local catalog. Synchronous (can take a
|
||||
while on a large library); a background/scheduled sync also runs periodically."""
|
||||
return plex_sync.sync(db)
|
||||
return {"ok": True, **plex_sync.sync(db)}
|
||||
|
||||
|
||||
def _link_dict(link: PlexLink | None) -> dict:
|
||||
@@ -129,7 +137,7 @@ def watch_link_set(
|
||||
admin token, so no separate Plex login. Enabling it for the first time runs the one-time
|
||||
"Plex is master" import immediately so the owner's existing Plex history shows up in Siftlode."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
raise HTTPException(status_code=409, detail="The Plex module is disabled.")
|
||||
raise refused(409, "plex_disabled")
|
||||
enabled = body.enabled
|
||||
link = db.execute(select(PlexLink).where(PlexLink.user_id == user.id)).scalars().first()
|
||||
if link is None:
|
||||
@@ -141,7 +149,7 @@ def watch_link_set(
|
||||
result: dict = {}
|
||||
if enabled and not link.initial_import_done:
|
||||
result = plex_watch.import_owner_watch_state(db, link)
|
||||
return {**_link_dict(link), "import": result or None}
|
||||
return {"ok": True, **_link_dict(link), "import": result or None}
|
||||
|
||||
|
||||
@router.post("/watch/import")
|
||||
@@ -150,6 +158,6 @@ def watch_import_now(user: User = Depends(admin_user), db: Session = Depends(get
|
||||
intersection; Siftlode-only states are preserved. Idempotent."""
|
||||
link = db.execute(select(PlexLink).where(PlexLink.user_id == user.id)).scalars().first()
|
||||
if link is None or not link.sync_enabled:
|
||||
raise HTTPException(status_code=409, detail="Enable Plex watch-sync first.")
|
||||
raise refused(409, "watch_sync_off")
|
||||
result = plex_watch.import_owner_watch_state(db, link)
|
||||
return {**_link_dict(link), "import": result}
|
||||
return {"ok": True, **_link_dict(link), "import": result}
|
||||
|
||||
@@ -9,6 +9,7 @@ from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -26,6 +27,10 @@ from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.plex.query import content_libs, runtime_by_key, top_art_keys
|
||||
from app.utils import like_escape
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# The most-restrictive → least ordering used to pick a collection's effective content rating from its
|
||||
# members (Plex shows one cert on the collection; we don't store it, so derive it).
|
||||
@@ -234,7 +239,7 @@ def collection_detail(
|
||||
endpoints below."""
|
||||
col = by_rating_key(db, PlexCollection, rating_key)
|
||||
if col is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown collection")
|
||||
raise not_found("collection")
|
||||
movies = (
|
||||
db.execute(
|
||||
select(PlexItem)
|
||||
@@ -325,9 +330,9 @@ def collection_detail(
|
||||
def _editable_collection_or_403(db: Session, rating_key: str) -> PlexCollection:
|
||||
col = by_rating_key(db, PlexCollection, rating_key)
|
||||
if col is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown collection")
|
||||
raise not_found("collection")
|
||||
if not _collection_editable(col):
|
||||
raise HTTPException(status_code=403, detail="This collection is read-only")
|
||||
raise refused(403, "collection_read_only")
|
||||
return col
|
||||
|
||||
|
||||
@@ -362,14 +367,13 @@ def _write_or_pending_error(wrote: bool, e: Exception) -> HTTPException:
|
||||
Both use 422, NOT a 5xx: the frontend treats 502/503/504 as "gateway unreachable" and raises a
|
||||
sticky offline banner (see api/core.ts RETRIABLE_GATEWAY). A 422 surfaces the detail as an ordinary
|
||||
action-error toast, which is what both of these are — the app answered, it didn't go unreachable."""
|
||||
if wrote:
|
||||
detail = (
|
||||
"Collection updated in Plex, but refreshing the local copy failed; "
|
||||
f"it will reconcile on the next sync ({e})"
|
||||
)
|
||||
else:
|
||||
detail = f"Plex write failed: {e}"
|
||||
return HTTPException(status_code=422, detail=detail)
|
||||
log.warning(
|
||||
"a Plex collection write did not settle",
|
||||
extra=logs.event(
|
||||
"plex.collections.write_failed", wrote=wrote, reason=type(e).__name__, error=str(e)
|
||||
),
|
||||
)
|
||||
return refused(422, "collection_pending_sync") if wrote else refused(422, "plex_write_failed")
|
||||
|
||||
|
||||
class CreateCollectionIn(BaseModel):
|
||||
@@ -389,7 +393,7 @@ def create_collection(
|
||||
title = body.title.strip()
|
||||
seed = body.item_rating_key
|
||||
if not title or not seed:
|
||||
raise HTTPException(status_code=400, detail="title and item_rating_key are required")
|
||||
raise refused(400, "collection_fields_required")
|
||||
if library:
|
||||
lib = (
|
||||
db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == library)).scalars().first()
|
||||
@@ -399,7 +403,7 @@ def create_collection(
|
||||
seed_item = by_rating_key(db, PlexItem, seed) or by_rating_key(db, PlexShow, seed)
|
||||
lib = db.get(PlexLibrary, seed_item.library_id) if seed_item else None
|
||||
if lib is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown library")
|
||||
raise not_found("library")
|
||||
_reject_duplicate_name(db, lib.id, title)
|
||||
wrote = False
|
||||
try:
|
||||
@@ -473,7 +477,7 @@ def collection_rename(
|
||||
col = _editable_collection_or_403(db, rating_key)
|
||||
title = body.title.strip()
|
||||
if not title:
|
||||
raise HTTPException(status_code=400, detail="title is required")
|
||||
raise refused(400, "title_required")
|
||||
_reject_duplicate_name(db, col.library_id, title, exclude_rk=rating_key)
|
||||
lib = db.get(PlexLibrary, col.library_id)
|
||||
wrote = False
|
||||
@@ -502,7 +506,7 @@ def collection_delete(
|
||||
db.commit()
|
||||
except (PlexError, PlexNotConfigured) as e:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=422, detail=f"Plex write failed: {e}")
|
||||
raise _write_or_pending_error(False, e) from None
|
||||
return {"deleted": rating_key}
|
||||
|
||||
|
||||
@@ -521,13 +525,10 @@ def collection_set_editable(
|
||||
(IMDb/TMDb/…) can never be made editable — they're regenerated outside Plex."""
|
||||
col = by_rating_key(db, PlexCollection, rating_key)
|
||||
if col is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown collection")
|
||||
raise not_found("collection")
|
||||
want = body.editable
|
||||
if want and (col.smart or _collection_source(col) != "collection"):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Smart / auto-list collections can't be made editable",
|
||||
)
|
||||
raise refused(400, "collection_smart")
|
||||
col.editable = want
|
||||
db.commit()
|
||||
return _collection_card(col)
|
||||
|
||||
@@ -5,7 +5,7 @@ import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -30,6 +30,7 @@ from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.routes.plex._shared import _norm_lang, item_or_404
|
||||
from app.routes.plex.collections import _collection_source
|
||||
from app.routes._common import not_found
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
@@ -109,7 +110,7 @@ def show_detail(
|
||||
hu = _norm_lang(lang) == "hu"
|
||||
sh = by_rating_key(db, PlexShow, rating_key)
|
||||
if sh is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex show")
|
||||
raise not_found("show")
|
||||
seasons = (
|
||||
db.execute(
|
||||
select(PlexSeason)
|
||||
|
||||
@@ -32,12 +32,12 @@ from app.plex.query import (
|
||||
meta_filter_conds,
|
||||
top_art_keys,
|
||||
)
|
||||
from app.discover.queries import RAIL_LIMIT
|
||||
from app.plex.tmdb import resolve_recs
|
||||
from app.routes.plex._shared import _norm_lang
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
RAIL_LIMIT = 20
|
||||
GENRE_TILES = 12
|
||||
SURPRISE_PER_KIND = 3 # Surprise-me: at most this many movies + this many shows (6 total)
|
||||
|
||||
|
||||
@@ -17,7 +17,8 @@ from fastapi import (
|
||||
)
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user, resolved_user_id
|
||||
@@ -38,6 +39,8 @@ from app.plex import stream as plex_stream
|
||||
from app.plex import watch_sync as plex_watch
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.routes.plex._shared import item_or_404
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
@@ -48,7 +51,7 @@ def _require_session(request: Request) -> int:
|
||||
low-sensitivity; a valid session is enough."""
|
||||
uid, _ = resolved_user_id(request)
|
||||
if not uid:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
raise refused(401, "not_authenticated")
|
||||
return uid
|
||||
|
||||
|
||||
@@ -77,15 +80,17 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response
|
||||
with SessionLocal() as db:
|
||||
key = imgcache.key_for(db, str(rating_key), v)
|
||||
if not key:
|
||||
raise HTTPException(status_code=404, detail="No image")
|
||||
raise not_found("image")
|
||||
plex = PlexClient(db) # reads base URL + token now; the fetch below needs no DB
|
||||
except PlexNotConfigured as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except PlexNotConfigured:
|
||||
raise refused(400, "plex_not_configured") from None
|
||||
try:
|
||||
with plex:
|
||||
data, ct = plex.image_bytes(key, w, h)
|
||||
except PlexError as e:
|
||||
raise HTTPException(status_code=502, detail=f"Plex image fetch failed: {e}")
|
||||
# silent-ok: a byte proxy for an <img>; the browser shows the placeholder and no reader sees the
|
||||
# detail — the 502 is the whole answer, as for the person-image proxy below.
|
||||
except PlexError:
|
||||
raise refused(502, "image_fetch_failed") from None
|
||||
imgcache.write(cache_key, data, ct)
|
||||
return imgcache.response(data, ct)
|
||||
|
||||
@@ -120,7 +125,7 @@ def subtitle_status(
|
||||
|
||||
`at` is the viewer's absolute position: the extractor windows around it, so what gets read first
|
||||
is the part actually about to be watched."""
|
||||
return subtitles.ensure(rating_key, sub_ord, at)
|
||||
return subtitles.ensure(rating_key, sub_ord, at, viewer=user.id)
|
||||
|
||||
|
||||
@router.get("/subtitle/{rating_key}/{sub_ord}")
|
||||
@@ -213,7 +218,7 @@ def person_image(u: str, request: Request) -> Response:
|
||||
Signed-session auth (no DB) so it, like /image, never holds a pool connection during the fetch."""
|
||||
_require_session(request)
|
||||
if not u.startswith(plex_people.PERSON_IMG_HOST):
|
||||
raise HTTPException(status_code=400, detail="Unsupported image host")
|
||||
raise refused(400, "image_host")
|
||||
cache_key = "person_" + hashlib.sha1(u.encode()).hexdigest()
|
||||
hit = imgcache.read(cache_key)
|
||||
if hit:
|
||||
@@ -223,7 +228,7 @@ def person_image(u: str, request: Request) -> Response:
|
||||
r = c.get(u)
|
||||
r.raise_for_status()
|
||||
except httpx.HTTPError:
|
||||
raise HTTPException(status_code=502, detail="Image fetch failed")
|
||||
raise refused(502, "image_fetch_failed")
|
||||
ct = r.headers.get("content-type", "image/jpeg")
|
||||
imgcache.write(cache_key, r.content, ct)
|
||||
return imgcache.response(r.content, ct)
|
||||
@@ -295,6 +300,37 @@ def _push_watch(
|
||||
)
|
||||
|
||||
|
||||
def _mark_finished(db: Session, user_id: int, item_id: int, now: datetime) -> bool:
|
||||
"""Mark the item watched (position cleared) and say whether THIS call made it so. Two near-end
|
||||
checkpoints arrive together routinely (the 10 s tick and the debounced `seeked` flush), and a
|
||||
read-then-write let both see "not watched" and both scrobble. The transition is decided by the
|
||||
write itself: an insert that did not conflict, or an update conditional on the old status —
|
||||
under READ COMMITTED the second writer re-reads the committed row and matches nothing."""
|
||||
values = {
|
||||
"status": "watched",
|
||||
"watched_at": now,
|
||||
"position_seconds": 0,
|
||||
"progress_updated_at": now,
|
||||
"synced_to_plex": False,
|
||||
}
|
||||
inserted = db.execute(
|
||||
pg_insert(PlexState)
|
||||
.values(user_id=user_id, item_id=item_id, **values)
|
||||
.on_conflict_do_nothing(constraint="uq_plex_user_item")
|
||||
.returning(PlexState.id)
|
||||
).first()
|
||||
if inserted is not None:
|
||||
return True
|
||||
mine = (PlexState.user_id == user_id) & (PlexState.item_id == item_id)
|
||||
if db.execute(
|
||||
update(PlexState).where(mine, PlexState.status != "watched").values(values)
|
||||
).rowcount:
|
||||
return True
|
||||
# Already watched: a re-finish still restamps the row, as it always did — just no scrobble.
|
||||
db.execute(update(PlexState).where(mine).values(values))
|
||||
return False
|
||||
|
||||
|
||||
class ItemProgressIn(BaseModel):
|
||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||
# one into a 500.
|
||||
@@ -330,11 +366,27 @@ def item_progress(
|
||||
# finish latch is up, or before the first position is known. Those are the states in which
|
||||
# reaping is the right answer anyway, and a viewer who scrubs back into a reaped session gets
|
||||
# the same 404 → reclaim the player already handles.
|
||||
plex_stream.touch_session(str(rating_key))
|
||||
plex_stream.touch_session(user.id, str(rating_key))
|
||||
position = max(0, int(body.position_seconds))
|
||||
duration = int(body.duration_seconds)
|
||||
final = body.final
|
||||
now = datetime.now(timezone.utc)
|
||||
near_end = duration > 0 and position > duration - _FINISH_MARGIN_S
|
||||
if near_end:
|
||||
flipped = _mark_finished(db, user.id, it.id, now)
|
||||
# The finish supersedes every resume push still in flight for this item: stamping it as the
|
||||
# latest push makes `_resume_push_superseded` drop them. Popping the entry instead (to keep
|
||||
# the map small) removed the very guard, and an older resume push could then land at Plex
|
||||
# AFTER the scrobble and revive a resume point on a watched item. Eviction bounds the map.
|
||||
with _push_throttle_lock:
|
||||
_last_push_at[(user.id, it.id)] = now
|
||||
db.commit()
|
||||
# Only the checkpoint that flipped the row scrobbles: a later checkpoint of the same finished
|
||||
# item — or a second one racing this one — would otherwise bump Plex's viewCount again.
|
||||
if flipped:
|
||||
_push_watch(background, db, user.id, it.id, rating_key, "watched")
|
||||
return {"ok": True, "status": "watched", "position_seconds": 0}
|
||||
|
||||
st = (
|
||||
db.execute(
|
||||
select(PlexState).where(PlexState.user_id == user.id, PlexState.item_id == it.id)
|
||||
@@ -342,28 +394,6 @@ def item_progress(
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
was_watched = st is not None and st.status == "watched"
|
||||
|
||||
near_end = duration > 0 and position > duration - _FINISH_MARGIN_S
|
||||
if near_end:
|
||||
if st is None:
|
||||
st = PlexState(user_id=user.id, item_id=it.id)
|
||||
db.add(st)
|
||||
st.status = "watched"
|
||||
st.watched_at = now
|
||||
st.position_seconds = 0
|
||||
st.progress_updated_at = now
|
||||
st.synced_to_plex = False
|
||||
# Terminal state for resume write-back: drop the throttle/ordering entry so the map doesn't
|
||||
# retain a dead key for every finished item.
|
||||
with _push_throttle_lock:
|
||||
_last_push_at.pop((user.id, it.id), None)
|
||||
db.commit()
|
||||
# Immediate on the watched transition; skip a repeat scrobble if it was already watched
|
||||
# (a later checkpoint of the same finished item would otherwise bump Plex's viewCount again).
|
||||
if not was_watched:
|
||||
_push_watch(background, db, user.id, it.id, rating_key, "watched")
|
||||
return {"status": "watched", "position_seconds": 0}
|
||||
|
||||
if position < _PROGRESS_MIN_S:
|
||||
if st is not None and st.status == "new":
|
||||
@@ -372,7 +402,7 @@ def item_progress(
|
||||
st.position_seconds = 0
|
||||
st.progress_updated_at = now
|
||||
db.commit()
|
||||
return {"position_seconds": 0}
|
||||
return {"ok": True, "position_seconds": 0}
|
||||
|
||||
if st is None:
|
||||
st = PlexState(user_id=user.id, item_id=it.id)
|
||||
@@ -397,7 +427,7 @@ def item_progress(
|
||||
duration * 1000,
|
||||
intent=now,
|
||||
)
|
||||
return {"position_seconds": position}
|
||||
return {"ok": True, "position_seconds": position}
|
||||
|
||||
|
||||
class ItemStateIn(BaseModel):
|
||||
@@ -418,7 +448,7 @@ def item_state(
|
||||
it = item_or_404(db, rating_key)
|
||||
status = body.status
|
||||
if status not in ("new", "watched", "hidden"):
|
||||
raise HTTPException(status_code=400, detail="Invalid status")
|
||||
raise refused(400, "invalid_status")
|
||||
st = (
|
||||
db.execute(
|
||||
select(PlexState).where(PlexState.user_id == user.id, PlexState.item_id == it.id)
|
||||
@@ -443,7 +473,7 @@ def item_state(
|
||||
db.commit()
|
||||
if mirrored_on_plex:
|
||||
_push_watch(background, db, user.id, it.id, rating_key, "unwatched")
|
||||
return {"status": "new"}
|
||||
return {"ok": True, "status": "new"}
|
||||
if st is None:
|
||||
st = PlexState(user_id=user.id, item_id=it.id)
|
||||
db.add(st)
|
||||
@@ -455,7 +485,7 @@ def item_state(
|
||||
db.commit()
|
||||
if status == "watched" and prev_status != "watched":
|
||||
_push_watch(background, db, user.id, it.id, rating_key, "watched")
|
||||
return {"status": status}
|
||||
return {"ok": True, "status": status}
|
||||
|
||||
|
||||
def _bulk_state(
|
||||
@@ -497,7 +527,9 @@ def _bulk_state(
|
||||
db.delete(st)
|
||||
if prev != "new":
|
||||
to_push.append((e.id, e.rating_key, "unwatched"))
|
||||
db.commit()
|
||||
# Flush, not commit: the handler commits, and it does so before the response — which is when
|
||||
# the background push below runs, so the push always reads the committed marks.
|
||||
db.flush()
|
||||
if pushable and to_push:
|
||||
# One coalesced background task for the whole batch (reuses a single session + Plex client),
|
||||
# not one task per episode — a 200-ep show mark is 1 task, not 200.
|
||||
@@ -520,7 +552,7 @@ def show_state(
|
||||
"""Mark a whole show watched/unwatched (all its episodes) for the current user + push to Plex."""
|
||||
sh = by_rating_key(db, PlexShow, rating_key)
|
||||
if sh is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex show")
|
||||
raise not_found("show")
|
||||
watched = body.watched
|
||||
eps = (
|
||||
db.execute(select(PlexItem).where(PlexItem.show_id == sh.id, PlexItem.kind == "episode"))
|
||||
@@ -528,7 +560,8 @@ def show_state(
|
||||
.all()
|
||||
)
|
||||
changed = _bulk_state(background, db, user.id, eps, watched)
|
||||
return {"changed": changed, "watched": watched}
|
||||
db.commit()
|
||||
return {"ok": True, "changed": changed, "watched": watched}
|
||||
|
||||
|
||||
@router.post("/season/{rating_key}/state")
|
||||
@@ -542,7 +575,7 @@ def season_state(
|
||||
"""Mark a whole season watched/unwatched (all its episodes) for the current user + push to Plex."""
|
||||
se = by_rating_key(db, PlexSeason, rating_key)
|
||||
if se is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex season")
|
||||
raise not_found("season")
|
||||
watched = body.watched
|
||||
eps = (
|
||||
db.execute(select(PlexItem).where(PlexItem.season_id == se.id, PlexItem.kind == "episode"))
|
||||
@@ -550,7 +583,8 @@ def season_state(
|
||||
.all()
|
||||
)
|
||||
changed = _bulk_state(background, db, user.id, eps, watched)
|
||||
return {"changed": changed, "watched": watched}
|
||||
db.commit()
|
||||
return {"ok": True, "changed": changed, "watched": watched}
|
||||
|
||||
|
||||
@router.post("/stream/{rating_key}/session")
|
||||
@@ -575,21 +609,23 @@ def stream_session(
|
||||
# the client can switch audio without a restart — even for otherwise direct-playable files.
|
||||
if it.playable == "direct" and audio is None and not aoff and not multi:
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "direct",
|
||||
"url": f"/api/plex/stream/{it.rating_key}/file",
|
||||
"start_s": 0.0,
|
||||
}
|
||||
if plex_paths.local_media_path(db, it.file_path) is None:
|
||||
raise HTTPException(status_code=404, detail="Media file not found on disk")
|
||||
s = plex_stream.start_session(db, it, start, audio, aoff, multi, transcode=transcode)
|
||||
raise refused(404, "media_missing")
|
||||
s = plex_stream.start_session(db, it, user.id, start, audio, aoff, multi, transcode=transcode)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Media file not found on disk")
|
||||
raise refused(404, "media_missing")
|
||||
# Report the REAL media start (the keyframe ffmpeg landed on, which hls.js zero-bases to), not the
|
||||
# requested offset — the client's absolute clock and the subtitle cue-shift both key off this.
|
||||
# `transcode`/`hw`/`tonemap` let the player show an honest cold-start ("Preparing stream…") and
|
||||
# surface the active mode + encoder in the stats overlay (`tonemap` = the HW HDR→SDR path); both
|
||||
# segment streams are otherwise identical.
|
||||
return {
|
||||
"ok": True,
|
||||
"mode": "hls",
|
||||
"start_s": s.media_start_s,
|
||||
# The session id is part of the path (not a query param) so the playlist's RELATIVE segment
|
||||
@@ -614,7 +650,7 @@ def stream_file(
|
||||
it = item_or_404(db, rating_key)
|
||||
src = plex_paths.local_media_path(db, it.file_path)
|
||||
if src is None:
|
||||
raise HTTPException(status_code=404, detail="Media file not found on disk")
|
||||
raise refused(404, "media_missing")
|
||||
if download:
|
||||
# A clean, safe download name from the title + the real file extension.
|
||||
ext = src.suffix or (f".{it.container}" if it.container else "")
|
||||
@@ -651,9 +687,9 @@ def stream_hls_file(
|
||||
references its segments/subtitles by relative name, all resolving back here.
|
||||
|
||||
The session id in the path is what makes "back here" mean the same session. Sessions are keyed by
|
||||
rating_key alone and a seek-restart or a second viewer of the same item replaces the registered
|
||||
one, while the DIRECTORY is per (key, offset, tag) — so resolving by rating_key handed the old
|
||||
viewer the new session's `seg_12.ts`, produced from a different offset under `-copyts`: wrong
|
||||
(viewer, rating_key) and a seek-restart replaces the viewer's registered one, while the DIRECTORY
|
||||
is per (key, viewer, offset, tag) — so resolving by the key alone handed the old player the new
|
||||
session's `seg_12.ts`, produced from a different offset under `-copyts`: wrong
|
||||
bytes with a 200, which the player has no way to notice (deep-review round 5). A request for a
|
||||
session that is no longer the registered one is a 404, and the player reclaims at the playhead.
|
||||
|
||||
@@ -668,10 +704,10 @@ def stream_hls_file(
|
||||
or "\\" in filename
|
||||
or ".." in filename
|
||||
):
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
s = plex_stream.current_session(str(rating_key))
|
||||
raise not_found("file")
|
||||
s = plex_stream.current_session(user.id, str(rating_key))
|
||||
if s is None or s.id != session_id:
|
||||
raise HTTPException(status_code=404, detail="No active playback session")
|
||||
raise refused(404, "session_gone")
|
||||
path = s.dir / filename
|
||||
# Only a name THIS session can actually produce is worth waiting for: its own playlists, and a
|
||||
# segment of one of its own renditions whose predecessor is already on disk (ffmpeg writes them
|
||||
@@ -681,14 +717,14 @@ def stream_hls_file(
|
||||
# past the produced region — or a deliberate flood of hopeless names — from parking threadpool
|
||||
# workers and their DB connections (see `artifact_may_arrive`).
|
||||
if not plex_stream.artifact_may_arrive(s, filename):
|
||||
raise HTTPException(status_code=404, detail="Not available yet")
|
||||
raise refused(404, "not_ready")
|
||||
timeout = 30 if ext == ".m3u8" else 20
|
||||
# `proc` so a session whose ffmpeg has already exited answers at once instead of polling out the
|
||||
# whole timeout: this is a SYNC route on the ~40-slot threadpool, holding a pooled DB connection
|
||||
# with it, so every such wait is a worker (and a connection) parked for 20 s — an ordinary seek
|
||||
# past the produced region was enough to do it (round 5).
|
||||
if not plex_stream.wait_for(path, timeout, proc=s.proc):
|
||||
raise HTTPException(status_code=404, detail="Not available yet")
|
||||
raise refused(404, "not_ready")
|
||||
try:
|
||||
content = path.read_bytes()
|
||||
except OSError:
|
||||
@@ -699,7 +735,7 @@ def stream_hls_file(
|
||||
# restarting at the playhead. Ordinary trigger: `loadSession` awaits the new session — which
|
||||
# kills and rmtrees the old one — before destroying the old hls instance, so the outgoing
|
||||
# instance's in-flight segment request races the rmtree on every seek-restart.
|
||||
raise HTTPException(status_code=404, detail="No active playback session") from None
|
||||
raise refused(404, "session_gone") from None
|
||||
return Response(
|
||||
content=content,
|
||||
media_type=_HLS_CT[ext],
|
||||
|
||||
@@ -2,9 +2,11 @@
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from fastapi import APIRouter, Depends
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user
|
||||
@@ -19,15 +21,12 @@ from app.models import (
|
||||
)
|
||||
from app.plex.cards import leaf_card
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.routes._common import owned_or_404
|
||||
from app.routes._common import owned_or_404, not_found
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
|
||||
def _own_playlist_or_404(db: Session, pid: int, user: User) -> PlexPlaylist:
|
||||
return owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
|
||||
|
||||
def _playlist_card(pl: PlexPlaylist, count: int, thumb_rk: str | None) -> dict:
|
||||
return {
|
||||
"id": pl.id,
|
||||
@@ -120,7 +119,7 @@ def create_playlist(
|
||||
) -> dict:
|
||||
title = body.title.strip()
|
||||
if not title:
|
||||
raise HTTPException(status_code=400, detail="title is required")
|
||||
raise refused(400, "title_required")
|
||||
pl = PlexPlaylist(user_id=user.id, title=title)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
@@ -140,7 +139,7 @@ def create_playlist(
|
||||
def playlist_detail(
|
||||
pid: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
rows = db.execute(_playlist_item_query(pid)).all()
|
||||
items = [it for (_pi, it) in rows]
|
||||
states = {
|
||||
@@ -183,10 +182,10 @@ def rename_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
title = body.title.strip()
|
||||
if not title:
|
||||
raise HTTPException(status_code=400, detail="title is required")
|
||||
raise refused(400, "title_required")
|
||||
pl.title = title
|
||||
db.commit()
|
||||
count = (
|
||||
@@ -203,7 +202,7 @@ def rename_playlist(
|
||||
def delete_playlist(
|
||||
pid: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
db.delete(pl)
|
||||
db.commit()
|
||||
return {"deleted": pid}
|
||||
@@ -220,10 +219,11 @@ def playlist_add_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
# Locked (like every add / reorder here): two adds otherwise read the same max position.
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
it = by_rating_key(db, PlexItem, body.item_rating_key)
|
||||
if it is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
raise not_found("item")
|
||||
exists = (
|
||||
db.execute(
|
||||
select(PlexPlaylistItem).where(
|
||||
@@ -252,7 +252,9 @@ def playlist_remove_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
# Locked like the adds and the reorder: the list row first, then its items — a remove that
|
||||
# locked an item row before a reorder locked the list deadlocked against it.
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
it = by_rating_key(db, PlexItem, item_rating_key)
|
||||
if it is not None:
|
||||
db.execute(
|
||||
@@ -288,16 +290,75 @@ def _items_by_rks(db: Session, rks: list) -> list[PlexItem]:
|
||||
return out
|
||||
|
||||
|
||||
class PlexItemAddBulkIn(PlexItemKeysIn):
|
||||
# Where each key goes back (E28 S3e2, the list page's "Undo" of a removal): an index over the
|
||||
# ORDERED rows, one per key. The two-call shape (append, then send the whole order) leaves the rows
|
||||
# at the end whenever the second call fails, and the order it sends would overwrite anything
|
||||
# another tab changed in between.
|
||||
positions: list[Annotated[int, Field(ge=0)]] | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _one_position_per_key(self):
|
||||
if self.positions is not None and len(self.positions) != len(self.item_rating_keys):
|
||||
raise ValueError("positions must pair one-to-one with item_rating_keys")
|
||||
return self
|
||||
|
||||
|
||||
def _insert_at(db: Session, pid: int, placed: list[tuple[int, PlexItem]]) -> int:
|
||||
"""Insert new rows at their indices (ascending, so each index is where the row stood when the
|
||||
rows before it were back), renumber the whole list 0..n-1, and say how many were added. An item
|
||||
already in the list does not move: the add stays idempotent."""
|
||||
rows = list(
|
||||
db.execute(
|
||||
select(PlexPlaylistItem)
|
||||
.where(PlexPlaylistItem.playlist_id == pid)
|
||||
.order_by(PlexPlaylistItem.position, PlexPlaylistItem.id)
|
||||
).scalars()
|
||||
)
|
||||
present = {r.item_id for r in rows}
|
||||
added = 0
|
||||
for at, it in sorted(placed, key=lambda p: p[0]):
|
||||
if it.id in present:
|
||||
continue
|
||||
row = PlexPlaylistItem(playlist_id=pid, item_id=it.id, position=0)
|
||||
db.add(row)
|
||||
rows.insert(min(at, len(rows)), row)
|
||||
present.add(it.id)
|
||||
added += 1
|
||||
for i, r in enumerate(rows):
|
||||
r.position = i
|
||||
return added
|
||||
|
||||
|
||||
@router.post("/playlists/{pid}/items/bulk")
|
||||
def playlist_add_bulk(
|
||||
pid: int,
|
||||
body: PlexItemKeysIn,
|
||||
body: PlexItemAddBulkIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Append many items at once (a whole season/show). Already-present items are skipped; the input
|
||||
order is preserved for the newly-added ones. Returns how many were added + the new total."""
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
order is preserved for the newly-added ones. With `positions`, each key is put back at its index
|
||||
instead of appended. Returns how many were added + the new total."""
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
if body.positions is not None:
|
||||
by_rk = {it.rating_key: it for it in _items_by_rks(db, body.item_rating_keys)}
|
||||
placed = [
|
||||
(at, by_rk[str(rk)])
|
||||
for rk, at in zip(body.item_rating_keys, body.positions, strict=True)
|
||||
if str(rk) in by_rk
|
||||
]
|
||||
added = _insert_at(db, pid, placed)
|
||||
if added:
|
||||
pl.updated_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
total = (
|
||||
db.scalar(
|
||||
select(func.count(PlexPlaylistItem.id)).where(PlexPlaylistItem.playlist_id == pid)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return {"ok": True, "added": added, "item_count": total}
|
||||
items = _items_by_rks(db, body.item_rating_keys)
|
||||
existing = set(
|
||||
db.execute(
|
||||
@@ -328,7 +389,7 @@ def playlist_add_bulk(
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return {"added": added, "item_count": total}
|
||||
return {"ok": True, "added": added, "item_count": total}
|
||||
|
||||
|
||||
@router.post("/playlists/{pid}/items/remove-bulk")
|
||||
@@ -340,7 +401,7 @@ def playlist_remove_bulk(
|
||||
) -> dict:
|
||||
"""Remove many items at once (a whole season/show, or a multi-select). Returns how many were
|
||||
removed + the new total."""
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
ids = [it.id for it in _items_by_rks(db, body.item_rating_keys)]
|
||||
removed = 0
|
||||
if ids:
|
||||
@@ -359,7 +420,7 @@ def playlist_remove_bulk(
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return {"removed": removed, "item_count": total}
|
||||
return {"ok": True, "removed": removed, "item_count": total}
|
||||
|
||||
|
||||
@router.put("/playlists/{pid}/order")
|
||||
@@ -370,7 +431,7 @@ def reorder_playlist(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Set the playlist order from an explicit list of item rating_keys (0-based positions)."""
|
||||
pl = _own_playlist_or_404(db, pid, user)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
order = body.item_rating_keys
|
||||
id_by_rk = {
|
||||
it.rating_key: it.id
|
||||
@@ -378,14 +439,27 @@ def reorder_playlist(
|
||||
select(PlexItem).where(PlexItem.rating_key.in_(order or [""]))
|
||||
).scalars()
|
||||
}
|
||||
for pos, rk in enumerate(order):
|
||||
iid = id_by_rk.get(rk)
|
||||
if iid is not None:
|
||||
db.execute(
|
||||
update(PlexPlaylistItem)
|
||||
.where(PlexPlaylistItem.playlist_id == pid, PlexPlaylistItem.item_id == iid)
|
||||
.values(position=pos)
|
||||
)
|
||||
rows = {
|
||||
r.item_id: r
|
||||
for r in db.execute(
|
||||
select(PlexPlaylistItem).where(PlexPlaylistItem.playlist_id == pid)
|
||||
).scalars()
|
||||
}
|
||||
pos = 0
|
||||
seen: set[int] = set()
|
||||
for rk in order:
|
||||
row = rows.get(id_by_rk.get(rk, -1))
|
||||
if row is not None and row.item_id not in seen:
|
||||
row.position = pos
|
||||
pos += 1
|
||||
seen.add(row.item_id)
|
||||
# Rows the payload omitted (added in another tab meanwhile) keep their relative order after
|
||||
# the given ones, so no two rows share a position (the YouTube reorder does the same).
|
||||
for row in sorted(
|
||||
(r for iid, r in rows.items() if iid not in seen), key=lambda r: (r.position, r.id)
|
||||
):
|
||||
row.position = pos
|
||||
pos += 1
|
||||
pl.updated_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
@@ -6,7 +6,7 @@ the app or any account data. Password-protected links exchange the password once
|
||||
a short-lived signed grant that the media URL carries (a `<video src>` can't send a header).
|
||||
"""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
@@ -19,6 +19,8 @@ from app.downloads import storage
|
||||
from app.models import DownloadJob, DownloadLink, MediaAsset
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import verify_password
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/public", tags=["public"])
|
||||
|
||||
@@ -37,7 +39,7 @@ _unlock_failures = RateLimiter(max_events=50, window_seconds=300)
|
||||
def _link_or_404(db: Session, token: str) -> DownloadLink:
|
||||
link = db.execute(select(DownloadLink).where(DownloadLink.token == token)).scalar_one_or_none()
|
||||
if link is None or linksmod.is_expired(link):
|
||||
raise HTTPException(status_code=404, detail="This link is no longer available.")
|
||||
raise refused(404, "link_gone")
|
||||
return link
|
||||
|
||||
|
||||
@@ -45,12 +47,12 @@ def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAs
|
||||
job = db.get(DownloadJob, link.job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
||||
if job is None or asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=404, detail="This video is no longer available.")
|
||||
raise refused(404, "video_gone")
|
||||
# Verify the file is actually on disk here (not just flagged ready), so the watch page never
|
||||
# loads a player for a missing file — meta and file agree.
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="This video is no longer available.")
|
||||
raise refused(404, "video_gone")
|
||||
return job, asset
|
||||
|
||||
|
||||
@@ -100,22 +102,24 @@ def watch_unlock(
|
||||
link = _link_or_404(db, token)
|
||||
viewer = f"{token}|{client_key(request)}"
|
||||
if _unlock_failures.exhausted(token) or not _unlock_limit.allow(viewer):
|
||||
raise HTTPException(status_code=429, detail="Too many attempts. Try again later.")
|
||||
raise refused(429, "too_many_attempts")
|
||||
if not link.password_hash:
|
||||
# Not a password link — nothing to unlock; just hand back the plain meta.
|
||||
job, asset = _ready_target(db, link)
|
||||
return {
|
||||
"ok": True,
|
||||
"needs_password": False,
|
||||
**linksmod.public_meta(link, job, asset, _file_url(token), _poster_url(token, asset)),
|
||||
}
|
||||
if not verify_password(body.password, link.password_hash):
|
||||
_unlock_failures.allow(token)
|
||||
raise HTTPException(status_code=403, detail="Wrong password.")
|
||||
raise refused(403, "wrong_share_password")
|
||||
job, asset = _ready_target(db, link)
|
||||
link.view_count += 1
|
||||
db.commit()
|
||||
grant = linksmod.make_grant(token, link.password_version)
|
||||
return {
|
||||
"ok": True,
|
||||
"needs_password": False,
|
||||
**linksmod.public_meta(
|
||||
link, job, asset, _file_url(token, grant), _poster_url(token, asset, grant)
|
||||
@@ -129,11 +133,11 @@ def watch_file(token: str, g: str | None = None, db: Session = Depends(get_db)):
|
||||
`allow_download` link serves it as an attachment. Password links require a valid grant."""
|
||||
link = _link_or_404(db, token)
|
||||
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
||||
raise HTTPException(status_code=403, detail="This link needs a password.")
|
||||
raise refused(403, "share_password_required")
|
||||
asset = _ready_asset(db, link)
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="This video is no longer available.")
|
||||
raise refused(404, "video_gone")
|
||||
|
||||
filename = storage.download_filename(asset.title or "video", asset.container, path)
|
||||
# The served Content-Type comes from the asset's ffprobe-verified media_kind + a container
|
||||
@@ -163,11 +167,11 @@ def watch_poster(token: str, g: str | None = None, db: Session = Depends(get_db)
|
||||
and the link-preview image). Password links require a valid grant, like the file."""
|
||||
link = _link_or_404(db, token)
|
||||
if link.password_hash and not linksmod.check_grant(token, g, link.password_version):
|
||||
raise HTTPException(status_code=403, detail="This link needs a password.")
|
||||
raise refused(403, "share_password_required")
|
||||
job, asset = _ready_target(db, link)
|
||||
if not asset.poster_path:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
@@ -8,6 +8,7 @@ from app.auth import require_human
|
||||
from app.db import get_db
|
||||
from app.models import SavedView, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.refusal import refused
|
||||
|
||||
# Saved "smart views": per-user named snapshots of a module's filters (feed or plex, keyed by
|
||||
# `source`). Gated behind require_human so the shared demo account can't pollute a communal list
|
||||
@@ -66,7 +67,7 @@ def create_view(
|
||||
) -> dict:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name is required")
|
||||
raise refused(400, "name_required")
|
||||
source = _norm_source(body.source)
|
||||
# Position is per (user, source), so each module's views number 0..N independently.
|
||||
maxpos = db.execute(
|
||||
@@ -86,14 +87,10 @@ def create_view(
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=409, detail=f"You already have a view named “{name}”.")
|
||||
raise refused(409, "view_name_taken", name=name)
|
||||
return _serialize(view)
|
||||
|
||||
|
||||
def _own_view(db: Session, user: User, view_id: int) -> SavedView:
|
||||
return owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
|
||||
|
||||
class UpdateViewIn(BaseModel):
|
||||
# PATCH: presence decides what's touched (read via model_fields_set). filters, when sent,
|
||||
# must be a real object — an explicit null is rejected (a view can't have null filters).
|
||||
@@ -109,16 +106,16 @@ def update_view(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
view = _own_view(db, user, view_id)
|
||||
view = owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name cannot be empty")
|
||||
raise refused(400, "name_required")
|
||||
view.name = name[:80]
|
||||
if "filters" in sent:
|
||||
if body.filters is None:
|
||||
raise HTTPException(status_code=400, detail="filters must be an object")
|
||||
raise refused(400, "filters_not_object")
|
||||
view.filters = body.filters
|
||||
if "is_default" in sent:
|
||||
make_default = bool(body.is_default)
|
||||
@@ -138,7 +135,7 @@ def update_view(
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=409, detail=f"You already have a view named “{view.name}”.")
|
||||
raise refused(409, "view_name_taken", name=view.name)
|
||||
return _serialize(view)
|
||||
|
||||
|
||||
@@ -148,7 +145,7 @@ def delete_view(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
view = _own_view(db, user, view_id)
|
||||
view = owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
db.delete(view)
|
||||
db.commit()
|
||||
return {"deleted": view_id}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
per-job run activity (from the in-process scheduler), the work still queued (DB-derived),
|
||||
and the shared quota picture."""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -25,6 +25,8 @@ from app.scheduler import (
|
||||
from app.sync.explore import followed_channel
|
||||
from app.sync.runner import estimate_deep_backfill
|
||||
from app.routes._common import not_found
|
||||
from app.refusal import refused
|
||||
from app.refusal import setting_refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin/scheduler", tags=["admin"])
|
||||
|
||||
@@ -46,10 +48,7 @@ def set_job_interval(
|
||||
raise not_found("job")
|
||||
minutes = body.interval_minutes
|
||||
if not (MIN_INTERVAL <= minutes <= MAX_INTERVAL):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"interval must be between {MIN_INTERVAL} and {MAX_INTERVAL} minutes",
|
||||
)
|
||||
raise refused(400, "invalid_interval", min=MIN_INTERVAL, max=MAX_INTERVAL)
|
||||
applied = apply_interval(db, job_id, minutes)
|
||||
audit.record(
|
||||
db,
|
||||
@@ -61,7 +60,7 @@ def set_job_interval(
|
||||
after={"interval_minutes": applied},
|
||||
)
|
||||
db.commit()
|
||||
return {"id": job_id, "interval_minutes": applied}
|
||||
return {"ok": True, "id": job_id, "interval_minutes": applied}
|
||||
|
||||
|
||||
@router.post("/jobs/{job_id}/run")
|
||||
@@ -77,14 +76,14 @@ def run_job_now(
|
||||
except KeyError:
|
||||
raise not_found("job")
|
||||
if result == "already_running":
|
||||
raise HTTPException(status_code=409, detail="Job is already running")
|
||||
return {"id": job_id, "started": True}
|
||||
raise refused(409, "job_running")
|
||||
return {"ok": True, "id": job_id, "started": True}
|
||||
|
||||
|
||||
@router.post("/run-all")
|
||||
def run_all_now(user: User = Depends(admin_user)) -> dict:
|
||||
"""Trigger every job that isn't already running. Returns the ids actually started."""
|
||||
return {"started": trigger_all(actor_id=user.id)}
|
||||
return {"ok": True, "started": trigger_all(actor_id=user.id)}
|
||||
|
||||
|
||||
class MaintenanceSettingsIn(BaseModel):
|
||||
@@ -102,8 +101,8 @@ def set_maintenance_settings(
|
||||
# set_value validates against the spec's min/max (one source of bounds), mirroring the config PATCH.
|
||||
try:
|
||||
sysconfig.set_value(db, "maintenance_revalidate_batch", body.revalidate_batch)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc) or "Invalid value")
|
||||
except sysconfig.InvalidSetting as exc:
|
||||
raise setting_refused(exc) from None
|
||||
new = sysconfig.get_int(db, "maintenance_revalidate_batch")
|
||||
audit.record(
|
||||
db,
|
||||
@@ -113,7 +112,7 @@ def set_maintenance_settings(
|
||||
after={"revalidate_batch": new},
|
||||
)
|
||||
db.commit()
|
||||
return {"revalidate_batch": new}
|
||||
return {"ok": True, "revalidate_batch": new}
|
||||
|
||||
|
||||
@router.get("")
|
||||
|
||||
@@ -16,7 +16,7 @@ dropped from the search response up front)."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, delete, func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
@@ -44,6 +44,8 @@ from app.sync.videos import insert_stubs, apply_video_details, parse_dt
|
||||
from app.youtube import search_scrape
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
from app.refusal import refused
|
||||
from app.routes._common import youtube_refusal
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
router = APIRouter(prefix="/api/search", tags=["search"])
|
||||
@@ -128,7 +130,7 @@ def _ingest_candidates(
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
|
||||
details = yt.get_channels(new_channel_ids)
|
||||
apply_channel_details(db, details)
|
||||
apply_channel_details(db, details, new_channel_ids)
|
||||
db.commit()
|
||||
except YouTubeError:
|
||||
# The stubs remain and the details can be backfilled later — but until they are, those
|
||||
@@ -258,7 +260,7 @@ def search_youtube(
|
||||
page, so it returns a single page (≤50) regardless. `cursor` lets the caller fetch further."""
|
||||
term = (q or "").strip()
|
||||
if not term:
|
||||
raise HTTPException(status_code=400, detail="A search term is required.")
|
||||
raise refused(400, "search_term_required")
|
||||
limit = max(1, min(limit, 100))
|
||||
|
||||
# Source: "scrape" (InnerTube, no API quota) or "api" (search.list, 100 units/page).
|
||||
@@ -275,20 +277,14 @@ def search_youtube(
|
||||
"a live search was refused: the per-user daily cap is spent",
|
||||
extra=logs.event("search.refused", reason="daily_cap", limit=daily_limit),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail=f"You've reached today's YouTube search limit ({daily_limit}). Try again tomorrow.",
|
||||
)
|
||||
raise refused(429, "search_daily_limit", limit=daily_limit)
|
||||
# The 100-unit budget pre-check only applies to the API source.
|
||||
if source == "api" and not quota.can_spend(db, SEARCH_COST):
|
||||
log.info(
|
||||
"a live search was refused: the shared YouTube budget is spent",
|
||||
extra=logs.event("search.refused", reason="shared_quota", cost=SEARCH_COST),
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="The shared YouTube quota for today is used up. Try again tomorrow.",
|
||||
)
|
||||
raise refused(429, "quota_spent")
|
||||
|
||||
# Channels this user has blocked are dropped before ingestion — never shown, never stored.
|
||||
blocked = set(
|
||||
@@ -313,9 +309,14 @@ def search_youtube(
|
||||
# dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY),
|
||||
# which hides the real message and burns a second quota-costed search. Upstream failure
|
||||
# on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1)
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"YouTube search failed: {exc}"
|
||||
) from exc
|
||||
# A Data API failure is logged where it happened; the scraper's is not, and its text
|
||||
# no longer rides the answer — so it is said here.
|
||||
if isinstance(exc, search_scrape.ScrapeError):
|
||||
log.warning(
|
||||
"the YouTube search scrape failed",
|
||||
extra=logs.event("search.scrape_failed", error=str(exc)),
|
||||
)
|
||||
raise youtube_refusal(exc) from exc
|
||||
|
||||
# Drop currently-live/upcoming, channel-less, and blocked-channel results.
|
||||
candidates = [
|
||||
@@ -357,9 +358,7 @@ def search_youtube(
|
||||
)
|
||||
if collected:
|
||||
break # keep what earlier pages already gathered
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"YouTube search failed: {exc}"
|
||||
) from exc
|
||||
raise youtube_refusal(exc) from exc
|
||||
next_cursor = page["next_page_token"]
|
||||
# The API source pages cost 100 units each — never auto-page; one page per request.
|
||||
if source == "api" or len(collected) >= limit or not next_cursor:
|
||||
@@ -412,7 +411,7 @@ def clear_search(
|
||||
find. Kept videos and organically-owned ones are left untouched."""
|
||||
ids = body.video_ids
|
||||
if not ids:
|
||||
return {"removed": 0}
|
||||
return {"ok": True, "removed": 0}
|
||||
db.execute(
|
||||
delete(SearchFind).where(SearchFind.user_id == user.id, SearchFind.video_id.in_(ids))
|
||||
)
|
||||
@@ -432,4 +431,4 @@ def clear_search(
|
||||
)
|
||||
).rowcount
|
||||
db.commit()
|
||||
return {"removed": removed}
|
||||
return {"ok": True, "removed": removed}
|
||||
|
||||
+14
-14
@@ -8,7 +8,7 @@ X-Setup-Token header).
|
||||
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException
|
||||
from fastapi import APIRouter, Depends, Header
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -21,6 +21,8 @@ from app.auth import PASSWORD_MIN_LEN
|
||||
from app.db import get_db
|
||||
from app.models import User
|
||||
from app.security import hash_password
|
||||
from app.refusal import refused
|
||||
from app.refusal import setting_refused
|
||||
|
||||
router = APIRouter(prefix="/api/setup", tags=["setup"])
|
||||
|
||||
@@ -31,9 +33,9 @@ def require_setup(
|
||||
) -> Session:
|
||||
"""Gate for setup steps. Returns the db session for the handler to reuse."""
|
||||
if state.is_configured(db):
|
||||
raise HTTPException(status_code=409, detail="This instance is already set up.")
|
||||
raise refused(409, "already_set_up")
|
||||
if not state.check_setup_token(db, x_setup_token):
|
||||
raise HTTPException(status_code=403, detail="Invalid or missing setup token.")
|
||||
raise refused(403, "setup_token")
|
||||
return db
|
||||
|
||||
|
||||
@@ -66,11 +68,9 @@ def setup_admin(body: SetupAdminIn, db: Session = Depends(require_setup)) -> dic
|
||||
email = body.email.strip().lower()
|
||||
password = body.password
|
||||
if "@" not in email:
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
if len(password) < PASSWORD_MIN_LEN:
|
||||
raise HTTPException(
|
||||
status_code=400, detail=f"Password must be at least {PASSWORD_MIN_LEN} characters."
|
||||
)
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
# Resolve canonically, not by the literal string (DX-17). The wizard can be re-run, and E9.5's
|
||||
# whole point is that two spellings of one Gmail mailbox are one person — matching literally here
|
||||
# would let a second run under the other spelling create the DUPLICATE admin row the epic exists
|
||||
@@ -97,16 +97,16 @@ def setup_config(settings_map: dict[str, Any], db: Session = Depends(require_set
|
||||
saved: list[str] = []
|
||||
for key, value in settings_map.items():
|
||||
if sysconfig.spec(key) is None:
|
||||
raise HTTPException(status_code=400, detail=f"Unknown setting: {key}")
|
||||
raise refused(400, "unknown_setting", key=key)
|
||||
if value is None or value == "":
|
||||
continue
|
||||
try:
|
||||
sysconfig.set_value(db, key, value)
|
||||
except (ValueError, RuntimeError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
except sysconfig.InvalidSetting as exc:
|
||||
raise setting_refused(exc) from None
|
||||
saved.append(key)
|
||||
db.commit() # sysconfig.set_value only flushes now — persist the wizard's config in one commit
|
||||
return {"saved": saved}
|
||||
return {"ok": True, "saved": saved}
|
||||
|
||||
|
||||
class SetupTestEmailIn(BaseModel):
|
||||
@@ -121,9 +121,9 @@ def setup_test_email(body: SetupTestEmailIn, db: Session = Depends(require_setup
|
||||
"""Send a test email to confirm the SMTP settings the wizard just saved."""
|
||||
to = body.to.strip()
|
||||
if "@" not in to:
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
if not email_mod.send_test(to, mailtext.normalize(body.lang)):
|
||||
raise HTTPException(status_code=400, detail="Couldn't send — check the SMTP settings.")
|
||||
raise refused(400, "smtp_failed")
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@@ -133,6 +133,6 @@ def setup_finish(db: Session = Depends(require_setup)) -> dict:
|
||||
invalidates the setup token and disables the wizard)."""
|
||||
has_admin = db.execute(select(User).where(User.role == "admin")).first() is not None
|
||||
if not has_admin:
|
||||
raise HTTPException(status_code=400, detail="Create the admin account first.")
|
||||
raise refused(400, "admin_first")
|
||||
state.mark_configured(db)
|
||||
return {"ok": True}
|
||||
|
||||
+16
-10
@@ -7,10 +7,13 @@ from app.audit import AuditAction
|
||||
from app.auth import admin_user, current_user, has_read_scope, require_human
|
||||
from app.db import get_db
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
from app.routes._common import youtube_refusal
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.sync.runner import estimate_deep_backfill
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
from app.scheduler import running_job_ids
|
||||
from app.youtube.client import YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/sync", tags=["sync"])
|
||||
|
||||
@@ -26,17 +29,20 @@ _sync_limiter = RateLimiter(max_events=3, window_seconds=600)
|
||||
def sync_subscriptions(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
if not has_read_scope(user):
|
||||
# No YouTube read grant yet — the onboarding wizard hasn't been completed.
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Connect your YouTube account (read access) to import subscriptions.",
|
||||
)
|
||||
raise refused(403, "youtube_read_scope")
|
||||
if not _sync_limiter.allow(str(user.id)):
|
||||
raise HTTPException(status_code=429, detail={"code": "sync_cooldown"})
|
||||
with quota.measured(db, user.id, quota.QuotaAction.SUBSCRIPTIONS_PULL) as m:
|
||||
result = import_subscriptions(db, user)
|
||||
try:
|
||||
with quota.measured(db, user.id, quota.QuotaAction.SUBSCRIPTIONS_PULL) as m:
|
||||
result = import_subscriptions(db, user)
|
||||
except YouTubeError as exc:
|
||||
# The onboarding wizard's first call: an unguarded failure here was a raw 500 and the
|
||||
# blocking "Server error" dialog, with nothing the user could act on (E9 F1).
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
result["quota_used_estimate"] = m.used
|
||||
result["quota_remaining_today"] = m.remaining
|
||||
return result
|
||||
return {"ok": True, **result}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
@@ -137,7 +143,7 @@ def deep_all(
|
||||
update(Subscription).where(Subscription.user_id == user.id).values(deep_requested=on)
|
||||
)
|
||||
db.commit()
|
||||
return {"updated": result.rowcount, "deep_requested": on}
|
||||
return {"ok": True, "updated": result.rowcount, "deep_requested": on}
|
||||
|
||||
|
||||
@router.post("/pause")
|
||||
@@ -145,7 +151,7 @@ def pause_sync(admin: User = Depends(admin_user), db: Session = Depends(get_db))
|
||||
state.set_sync_paused(db, True)
|
||||
audit.record(db, admin.id, AuditAction.SYNC_PAUSE, summary="Background sync paused")
|
||||
db.commit()
|
||||
return {"paused": True}
|
||||
return {"ok": True, "paused": True}
|
||||
|
||||
|
||||
@router.post("/resume")
|
||||
@@ -153,4 +159,4 @@ def resume_sync(admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
state.set_sync_paused(db, False)
|
||||
audit.record(db, admin.id, AuditAction.SYNC_RESUME, summary="Background sync resumed")
|
||||
db.commit()
|
||||
return {"paused": False}
|
||||
return {"ok": True, "paused": False}
|
||||
|
||||
+24
-14
@@ -1,4 +1,4 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
@@ -9,6 +9,7 @@ from app.db import get_db
|
||||
from app.models import ChannelTag, Tag, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.sync.tagsuggest import compute_tag_suggestions
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/tags", tags=["tags"])
|
||||
|
||||
@@ -60,7 +61,7 @@ def create_tag(
|
||||
) -> dict:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name is required")
|
||||
raise refused(400, "name_required")
|
||||
# An unknown category silently falls back to "other" (unchanged behaviour) rather than 422.
|
||||
category = body.category if body.category in ("language", "topic", "other") else "other"
|
||||
tag = Tag(
|
||||
@@ -74,13 +75,14 @@ def create_tag(
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=409, detail=f"You already have a tag named “{name}”.")
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color, "category": tag.category}
|
||||
|
||||
|
||||
def _own_tag(db: Session, user: User, tag_id: int) -> Tag:
|
||||
# System tags (user_id NULL) are not user-editable → owned_or_404 404s them too.
|
||||
return owned_or_404(db, Tag, tag_id, user, what="tag")
|
||||
raise refused(409, "tag_name_taken", name=name)
|
||||
return {
|
||||
"ok": True,
|
||||
"id": tag.id,
|
||||
"name": tag.name,
|
||||
"color": tag.color,
|
||||
"category": tag.category,
|
||||
}
|
||||
|
||||
|
||||
class TagUpdateIn(BaseModel):
|
||||
@@ -97,12 +99,13 @@ def update_tag(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
tag = _own_tag(db, user, tag_id)
|
||||
# System tags (user_id NULL) are not user-editable: owned_or_404 404s them too.
|
||||
tag = owned_or_404(db, Tag, tag_id, user, what="tag")
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name cannot be empty")
|
||||
raise refused(400, "name_required")
|
||||
tag.name = name
|
||||
if "color" in sent:
|
||||
tag.color = body.color
|
||||
@@ -110,8 +113,14 @@ def update_tag(
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=409, detail=f"You already have a tag named “{tag.name}”.")
|
||||
return {"id": tag.id, "name": tag.name, "color": tag.color, "category": tag.category}
|
||||
raise refused(409, "tag_name_taken", name=tag.name)
|
||||
return {
|
||||
"ok": True,
|
||||
"id": tag.id,
|
||||
"name": tag.name,
|
||||
"color": tag.color,
|
||||
"category": tag.category,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{tag_id}")
|
||||
@@ -120,7 +129,8 @@ def delete_tag(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
tag = _own_tag(db, user, tag_id)
|
||||
# System tags (user_id NULL) are not user-editable: owned_or_404 404s them too.
|
||||
tag = owned_or_404(db, Tag, tag_id, user, what="tag")
|
||||
db.delete(tag) # ChannelTag rows cascade on tag delete.
|
||||
db.commit()
|
||||
return {"deleted": tag_id}
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Callable
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import audit, logs, logsink, progress, quota
|
||||
from app import audit, debugsession, logs, logsink, progress, quota
|
||||
from app.audit import AuditAction
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
@@ -386,9 +386,13 @@ def _log_gc_job() -> None:
|
||||
def work(db):
|
||||
from app import sysconfig
|
||||
|
||||
reaped = logsink.gc(db, sysconfig.get_int(db, "log_retention_days"))
|
||||
days = sysconfig.get_int(db, "log_retention_days")
|
||||
reaped = logsink.gc(db, days)
|
||||
# Debug sessions (E16) go with the rows they bundle; an open one past its cap is closed.
|
||||
closed = debugsession.sweep(db)
|
||||
sessions = debugsession.gc(db, days)
|
||||
db.commit()
|
||||
return {"reaped": reaped}
|
||||
return {"reaped": reaped, "sessions_closed": closed, "sessions_reaped": sessions}
|
||||
|
||||
_job("log_gc", work)
|
||||
|
||||
|
||||
+185
-31
@@ -8,13 +8,15 @@ ingested (with a stub channel) so the mirror is faithful."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import TypeVar
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import Engine, create_engine, delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app import logs, progress, quota
|
||||
from app.auth import has_read_scope
|
||||
@@ -27,6 +29,87 @@ log = logs.get_logger(__name__)
|
||||
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
||||
WRITE_UNIT_COST = 50
|
||||
|
||||
# The advisory-lock namespaces ("PL" per playlist, "PU" per user), so a key cannot meet another
|
||||
# feature's lock on the same number (Postgres keeps the two-int keys apart from the one-bigint ones).
|
||||
_SYNC_LOCK_NS = 0x504C
|
||||
_USER_SYNC_LOCK_NS = 0x5055
|
||||
|
||||
# The claims' own connections, outside the app pool: a claim is held for a whole push (seconds of
|
||||
# YouTube calls) beside the request's pooled session and quota's, so a burst of pushes drawing a
|
||||
# third pooled connection each ran the pool dry and stalled every other request for pool_timeout.
|
||||
# NullPool opens one per claim and closes it after, so a lock can never ride back into a pool either.
|
||||
_claim_engines: dict[str, Engine] = {}
|
||||
|
||||
|
||||
def _claim_engine(db: Session) -> Engine:
|
||||
url = db.get_bind().engine.url
|
||||
key = url.render_as_string(hide_password=False)
|
||||
eng = _claim_engines.get(key)
|
||||
if eng is None:
|
||||
eng = _claim_engines[key] = create_engine(url, poolclass=NullPool, hide_parameters=True)
|
||||
return eng
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _advisory_claim(db: Session, ns: int, key: int) -> Iterator[bool]:
|
||||
"""Try a session-level advisory lock for the block on a connection of its own; yields whether it
|
||||
was free. It survives the block's commits, and a crashed worker's lock dies with its connection
|
||||
— no claim column that a crash leaves stuck."""
|
||||
conn = _claim_engine(db).connect()
|
||||
held = False
|
||||
try:
|
||||
held = bool(conn.scalar(select(func.pg_try_advisory_lock(ns, key))))
|
||||
conn.commit()
|
||||
yield held
|
||||
finally:
|
||||
try:
|
||||
if held:
|
||||
conn.scalar(select(func.pg_advisory_unlock(ns, key)))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def sync_claim(db: Session, playlist_id: int) -> AbstractContextManager[bool]:
|
||||
"""Claim one playlist's YouTube sync (push, re-pull, the scheduled mirror refresh) for the block;
|
||||
yields whether it was free. A push reads YouTube, then writes both sides for seconds and commits
|
||||
step by step, so a row lock ends at its first commit and two tabs (or a push and the scheduler)
|
||||
interleave into a doubled YouTube item or a baseline recorded over the other's writes."""
|
||||
return _advisory_claim(db, _SYNC_LOCK_NS, playlist_id)
|
||||
|
||||
|
||||
class PlaylistGone(Exception):
|
||||
"""The list a sync was working on was deleted meanwhile (another tab, the mirror drop). Its
|
||||
items went with it, so a plan made now reads "every item removed here" — and would delete them
|
||||
all from the YouTube playlist the reader chose to keep. The route answers 404."""
|
||||
|
||||
|
||||
def lock_playlists(db: Session, *playlist_ids: int) -> set[int]:
|
||||
"""Row-lock playlists before touching their items, several in id order. Every item write takes
|
||||
the PLAYLIST first — the route edits, a push, a re-pull, the scheduled mirror refresh and a
|
||||
delete alike: a writer that locked an item row first and the playlist second (a push's item
|
||||
replace beside a remove) deadlocks against one that went the other way, and the loser is a
|
||||
500. The rows are re-read, so `dirty`, the name and the sync baseline are the ones a finished
|
||||
writer committed. Held to the caller's next commit — so no commit may come between this and
|
||||
the write it guards. Returns the ids it found: a list deleted meanwhile is not among them."""
|
||||
rows = db.scalars(
|
||||
select(Playlist)
|
||||
.where(Playlist.id.in_(sorted(set(playlist_ids))))
|
||||
.order_by(Playlist.id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).all() # consumed: rows fetched are what refreshes the loaded objects
|
||||
return {pl.id for pl in rows}
|
||||
|
||||
|
||||
def _lock_or_gone(db: Session, pl: Playlist) -> None:
|
||||
"""`lock_playlists` for a sync that goes on to write YouTube or the items: a list deleted since
|
||||
it was read raises `PlaylistGone` (rolled back) instead of being planned as an empty one."""
|
||||
pl_id = pl.id # read first: the rollback expires `pl`, and its row is gone
|
||||
if pl_id not in lock_playlists(db, pl_id):
|
||||
db.rollback()
|
||||
raise PlaylistGone(pl_id)
|
||||
|
||||
|
||||
def fingerprint(name: str, ids: list[str]) -> str:
|
||||
"""A stable checksum of a playlist's name + ordered video ids. Stored as the synced
|
||||
@@ -91,11 +174,18 @@ def parse_unavailable(pl: Playlist) -> set[str]:
|
||||
return _load_baseline(pl)[1]
|
||||
|
||||
|
||||
def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None = None) -> None:
|
||||
def _set_baseline(
|
||||
pl: Playlist,
|
||||
ids: list[str],
|
||||
unavailable: list[str] | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
) -> None:
|
||||
"""Record `ids` as both the hash (drives `dirty`, over the local ids only) and the raw ordered
|
||||
set (drives merge). `unavailable` = un-ingestable-but-live YouTube ids to carry forward so the
|
||||
next merge skips them; stored only when non-empty (keeps the common case a plain list)."""
|
||||
pl.synced_fingerprint = fingerprint(pl.name, ids)
|
||||
next merge skips them; stored only when non-empty (keeps the common case a plain list). `name`:
|
||||
the title YouTube now has, when that may differ from the list's own (see the push)."""
|
||||
pl.synced_fingerprint = fingerprint(pl.name if name is None else name, ids)
|
||||
if unavailable:
|
||||
pl.synced_video_ids = json.dumps({"ids": ids, "unavailable": list(unavailable)})
|
||||
else:
|
||||
@@ -103,6 +193,25 @@ def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None =
|
||||
pl.dirty = False
|
||||
|
||||
|
||||
def _set_baseline_under_lock(
|
||||
db: Session,
|
||||
pl: Playlist,
|
||||
ids: list[str],
|
||||
unavailable: list[str] | None = None,
|
||||
*,
|
||||
name: str,
|
||||
) -> None:
|
||||
"""A push's closing baseline: `ids` and `name` are what YouTube now holds. The push spent
|
||||
seconds on YouTube writes since it replaced the local items, so an edit may have landed
|
||||
meanwhile — lock the list and derive `dirty` from what is there, or the edit would read as
|
||||
synced and never reach YouTube. The name too: the lock re-reads a rename made meanwhile, and a
|
||||
baseline taken over THAT name matched it, so the list read clean with YouTube's title still the
|
||||
old one (E28 review round 4)."""
|
||||
_lock_or_gone(db, pl)
|
||||
_set_baseline(pl, ids, unavailable, name=name)
|
||||
recompute_dirty(db, pl)
|
||||
|
||||
|
||||
def recompute_dirty(db: Session, pl: Playlist) -> None:
|
||||
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
|
||||
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
|
||||
@@ -159,15 +268,13 @@ def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None
|
||||
db.commit()
|
||||
|
||||
|
||||
def _replace_items_from_youtube(
|
||||
db: Session, yt: YouTubeClient, pl: Playlist, ids: list[str]
|
||||
) -> list[str]:
|
||||
def _replace_items_from_youtube(db: Session, pl: Playlist, ids: list[str]) -> list[str]:
|
||||
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
|
||||
replace): ensure the videos exist in the catalog, keep only those present in their YT
|
||||
order (de-duplicated), swap the items out, and reset the synced fingerprint + dirty flag.
|
||||
`pl.name` must already be set to the desired name. The caller commits. Returns the final
|
||||
ordered ids."""
|
||||
_ensure_videos(db, yt, ids)
|
||||
replace): keep only those present in the catalog in their YT order (de-duplicated), swap the
|
||||
items out, and reset the synced fingerprint + dirty flag. The caller has ingested `ids`
|
||||
(`_ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
|
||||
after: an ingest in here re-fetched every id YouTube will not return (a private or deleted
|
||||
video) and its commit let go of the lock before the replace. Returns the final ordered ids."""
|
||||
present = (
|
||||
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
|
||||
)
|
||||
@@ -196,6 +303,21 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
||||
the user's local playlists and the built-in Watch later untouched."""
|
||||
if not has_read_scope(user):
|
||||
return {"synced": 0, "reason": "no read scope"}
|
||||
with user_sync_claim(db, user.id) as free:
|
||||
if not free:
|
||||
return {"synced": 0, "reason": "busy"}
|
||||
return pull_user_playlists(db, user)
|
||||
|
||||
|
||||
def user_sync_claim(db: Session, user_id: int) -> AbstractContextManager[bool]:
|
||||
"""Claim a user's bulk mirror pull for the block; yields whether it was free. Two runs at once
|
||||
(the scheduler beside a manual "sync now", two tabs) both saw a YouTube list unmirrored and
|
||||
each created its mirror — a duplicate row."""
|
||||
return _advisory_claim(db, _USER_SYNC_LOCK_NS, user_id)
|
||||
|
||||
|
||||
def pull_user_playlists(db: Session, user: User) -> dict:
|
||||
"""The mirror pull itself; the caller holds `user_sync_claim`."""
|
||||
synced = 0
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt_playlists = list(yt.iter_my_playlists())
|
||||
@@ -219,11 +341,15 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
# Drop mirrors whose YouTube playlist no longer exists.
|
||||
for ytid, pl in existing.items():
|
||||
if ytid not in yt_ids:
|
||||
db.delete(pl)
|
||||
db.flush()
|
||||
# Drop mirrors whose YouTube playlist no longer exists — locked first, as every item write.
|
||||
gone = [pl for ytid, pl in existing.items() if ytid not in yt_ids]
|
||||
if gone:
|
||||
lock_playlists(db, *(pl.id for pl in gone))
|
||||
for pl in gone:
|
||||
db.delete(pl)
|
||||
# Committed now: a run where no list below commits (all dirty, or all claimed) would
|
||||
# otherwise drop the deletions with the session.
|
||||
db.commit()
|
||||
|
||||
for idx, p in enumerate(yt_playlists):
|
||||
ytid = p.get("id")
|
||||
@@ -245,12 +371,27 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
||||
)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
else:
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
# The user is pushing or re-pulling this very list: it writes the same rows.
|
||||
# The next scheduled run refreshes it.
|
||||
continue
|
||||
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
||||
# Ingest first — it commits — so the row lock below is held from the dirty check
|
||||
# to the commit that replaces the items.
|
||||
_ensure_videos(db, yt, ids)
|
||||
# `dirty` read above is a snapshot from before the YouTube calls (sessions keep
|
||||
# loaded rows past a commit): a remove or reorder made meanwhile must not be
|
||||
# replaced by YouTube's order. Item writes lock the playlist row, so this waits for
|
||||
# one in flight and sees what it committed.
|
||||
lock_playlists(db, pl.id)
|
||||
if pl.dirty:
|
||||
db.rollback()
|
||||
continue
|
||||
pl.name = p.get("title") or pl.name
|
||||
# Mirror is authoritative: replace the items to match YouTube's order.
|
||||
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
||||
_replace_items_from_youtube(db, yt, pl, ids)
|
||||
db.commit()
|
||||
# Mirror is authoritative: replace the items to match YouTube's order.
|
||||
_replace_items_from_youtube(db, pl, ids)
|
||||
db.commit()
|
||||
synced += 1
|
||||
return {"synced": synced}
|
||||
|
||||
@@ -488,7 +629,8 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
breaker = _Breaker(playlist_id=pl.id)
|
||||
with YouTubeClient(db, user) as yt:
|
||||
if not pl.yt_playlist_id:
|
||||
pl.yt_playlist_id = yt.create_playlist(pl.name)
|
||||
pushed_name = pl.name
|
||||
pl.yt_playlist_id = yt.create_playlist(pushed_name)
|
||||
db.commit()
|
||||
created = 1
|
||||
# Brand-new playlist: just append every local video in order.
|
||||
@@ -501,7 +643,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
if not isinstance(added, _Failed):
|
||||
inserted += 1
|
||||
if not failures:
|
||||
_set_baseline(pl, local)
|
||||
_set_baseline_under_lock(db, pl, local, name=pushed_name)
|
||||
db.commit()
|
||||
return {
|
||||
"created": created,
|
||||
@@ -542,6 +684,14 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
||||
# YouTube write trips the breaker.
|
||||
_ensure_videos(db, yt, mp.add_to_local)
|
||||
# `local` was read before the YouTube calls above: an add or remove committed meanwhile
|
||||
# (another tab) would be undone by the replace below. Lock the list — the ingest's commit is
|
||||
# behind us — and plan again from what is there now. Nothing new needs ingesting: a video
|
||||
# the re-read adds is already a local row. A list deleted meanwhile stops here, before a plan
|
||||
# over its now-empty items removes them all from YouTube.
|
||||
_lock_or_gone(db, pl)
|
||||
local = _local_video_ids(db, pl)
|
||||
mp = merge_plan(parse_baseline(pl), local, yt_ids, effective_unavailable)
|
||||
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
|
||||
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
|
||||
present = (
|
||||
@@ -555,10 +705,12 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
_set_local_items(db, pl, final_local)
|
||||
db.commit()
|
||||
|
||||
# --- YouTube side (metered, breaker-guarded).
|
||||
if snippet is not None and (snippet.get("title") or "") != pl.name:
|
||||
# --- YouTube side (metered, breaker-guarded). The name as of the lock above: the closing
|
||||
# baseline records THIS one, whatever a rename made during the writes below says.
|
||||
pushed_name = pl.name
|
||||
if snippet is not None and (snippet.get("title") or "") != pushed_name:
|
||||
try:
|
||||
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
|
||||
yt.update_playlist_title(pl.yt_playlist_id, pushed_name)
|
||||
except YouTubeError as exc:
|
||||
# All the caller passes on is the word "title", which says the rename did not
|
||||
# happen but not that YouTube refused it, or why.
|
||||
@@ -626,7 +778,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
(effective_unavailable | {v for v in mp.add_to_local if v not in present})
|
||||
& yt_id_set
|
||||
)
|
||||
_set_baseline(pl, final_local, unavailable)
|
||||
_set_baseline_under_lock(db, pl, final_local, unavailable, name=pushed_name)
|
||||
db.commit()
|
||||
return {
|
||||
"created": created,
|
||||
@@ -652,9 +804,11 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||
if snippet is None:
|
||||
raise YouTubeError("Playlist no longer exists on YouTube")
|
||||
pl.name = snippet.get("title") or pl.name
|
||||
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
||||
ordered = _replace_items_from_youtube(db, yt, pl, ids)
|
||||
_ensure_videos(db, yt, ids)
|
||||
_lock_or_gone(db, pl)
|
||||
pl.name = snippet.get("title") or pl.name
|
||||
ordered = _replace_items_from_youtube(db, pl, ids)
|
||||
db.commit()
|
||||
return {"items": len(ordered)}
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
@@ -47,8 +48,37 @@ def _external_links(branding: dict) -> list | None:
|
||||
return out or None
|
||||
|
||||
|
||||
def apply_channel_details(db: Session, items: list[dict]) -> None:
|
||||
# How old a channel's metadata may get before it is fetched again. Banners, avatars, descriptions and
|
||||
# counts change on YouTube, and an old banner URL stops resolving at all — without a refresh a channel
|
||||
# kept whatever it had the day it was first seen. A week keeps counts roughly current at almost no
|
||||
# cost: `channels.list` is 1 quota unit per 50 channels, and only the stale ones are asked for.
|
||||
DETAILS_MAX_AGE = timedelta(days=7)
|
||||
|
||||
|
||||
def details_stale(channel: Channel, now: datetime) -> bool:
|
||||
"""Whether `channel`'s metadata is missing or older than DETAILS_MAX_AGE (see `details_stale_clause`)."""
|
||||
synced = channel.details_synced_at
|
||||
return synced is None or synced < now - DETAILS_MAX_AGE
|
||||
|
||||
|
||||
def details_stale_clause(now: datetime):
|
||||
"""`details_stale` as a WHERE clause, so the batch import and a single channel agree on "stale"."""
|
||||
return or_(
|
||||
Channel.details_synced_at.is_(None),
|
||||
Channel.details_synced_at < now - DETAILS_MAX_AGE,
|
||||
)
|
||||
|
||||
|
||||
def apply_channel_details(db: Session, items: list[dict], asked: Iterable[str]) -> None:
|
||||
"""Write `channels.list`'s answer onto the rows. `asked` is every id the call asked for: one
|
||||
YouTube did not return (terminated, deleted) is stamped as synced too — otherwise it stays
|
||||
stale forever and every view / import asks for it again, spending quota on a known miss."""
|
||||
now = datetime.now(timezone.utc)
|
||||
returned = {item["id"] for item in items}
|
||||
for cid in set(asked) - returned:
|
||||
missing = db.get(Channel, cid)
|
||||
if missing is not None:
|
||||
missing.details_synced_at = now
|
||||
for item in items:
|
||||
channel = db.get(Channel, item["id"])
|
||||
if channel is None:
|
||||
@@ -92,7 +122,7 @@ def should_prune(fetched_count: int, existing_count: int) -> bool:
|
||||
def import_subscriptions(db: Session, user: User) -> dict:
|
||||
"""Pull all subscriptions for `user`, upsert channels + subscription links, prune
|
||||
channels the user has unsubscribed from on YouTube, and fetch channel details for
|
||||
any channel we don't have details for yet."""
|
||||
any channel whose details are missing or stale (`DETAILS_MAX_AGE`)."""
|
||||
fetched: dict[str, dict] = {}
|
||||
new_channels = 0
|
||||
|
||||
@@ -145,12 +175,12 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
||||
removed += 1
|
||||
db.commit()
|
||||
|
||||
# Fetch details for channels that don't have them yet.
|
||||
# Fetch details for channels that don't have them yet, or had them too long ago.
|
||||
need_details = (
|
||||
db.execute(
|
||||
select(Channel).where(
|
||||
Channel.id.in_(fetched.keys()),
|
||||
Channel.details_synced_at.is_(None),
|
||||
details_stale_clause(datetime.now(timezone.utc)),
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
@@ -158,8 +188,9 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
||||
)
|
||||
detailed = 0
|
||||
if need_details:
|
||||
items = yt.get_channels([c.id for c in need_details])
|
||||
apply_channel_details(db, items)
|
||||
ids = [c.id for c in need_details]
|
||||
items = yt.get_channels(ids)
|
||||
apply_channel_details(db, items, ids)
|
||||
detailed = len(items)
|
||||
db.commit()
|
||||
|
||||
|
||||
+80
-16
@@ -137,10 +137,29 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
min=1,
|
||||
max=100_000,
|
||||
),
|
||||
# min=1, NOT 0: the per-file cap is enforced as `if cap and size > cap`, so a stored 0 would read
|
||||
# as falsy and SILENTLY disable the limit (an unbounded streamed write to the volume). Floor it at
|
||||
# 1 so the cap is always active; the overall footprint is separately bounded by the byte quota.
|
||||
ConfigSpec("download_upload_max_bytes", "int", "downloads", "download_upload_max_bytes", min=1),
|
||||
# min=1, NOT 0: a stored 0 used to read as "no per-file cap" (the cap was enforced as
|
||||
# `if cap and size > cap`, so a falsy value disabled it). Since P1 the route passes this number
|
||||
# straight to `capped_stream_to`, where 0 would refuse every upload instead — the opposite
|
||||
# failure, and the reason `config.download_upload_max_bytes` carries its own `ge=1`. Either way
|
||||
# a stored 0 is wrong, so it is floored at 1; the overall footprint is separately bounded by the
|
||||
# byte quota.
|
||||
# `max` is not cosmetic: `bodylimit.ROUTE_MAX_BODY_BYTES` refuses an upload body over exactly
|
||||
# this number at the edge, before the route's own check ever sees it. An admin allowed to raise
|
||||
# the knob past the edge would get 413s from a layer the admin page never mentions, so the two
|
||||
# are pinned together here and `tests/test_body_limit.py` holds them that way. Below the edge is
|
||||
# the useful direction and the one the knob exists for: a lowered knob is what makes the route's
|
||||
# own 422 — the one carrying this figure — the answer the user gets. `_coerce` also
|
||||
# REJECTS a stored value outside these bounds on read (falling back to the env default, with a
|
||||
# warning) — neither bound has always been what it is now, so a legal-when-written override can
|
||||
# be sitting in the database already.
|
||||
ConfigSpec(
|
||||
"download_upload_max_bytes",
|
||||
"int",
|
||||
"downloads",
|
||||
"download_upload_max_bytes",
|
||||
min=1,
|
||||
max=settings.download_upload_max_bytes,
|
||||
),
|
||||
ConfigSpec(
|
||||
"download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650
|
||||
),
|
||||
@@ -176,6 +195,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"),
|
||||
# (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.)
|
||||
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
|
||||
ConfigSpec("plex_max_sessions", "int", "plex", "plex_max_sessions", min=1, max=64),
|
||||
# Transcode re-encode tuning (E6 S1). The libx264 knobs apply only to the software fallback.
|
||||
# crf 0–51 (x264 range); max_height 0 = source.
|
||||
ConfigSpec("plex_transcode_hw", "choice", "plex", "plex_transcode_hw"),
|
||||
@@ -191,6 +211,10 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
# --- TMDB enrichment (E5 S2; optional) — the API key is the only secret ---
|
||||
ConfigSpec("tmdb_api_key", "str", "tmdb", "tmdb_api_key", secret=True),
|
||||
ConfigSpec("tmdb_language", "str", "tmdb", "tmdb_language"),
|
||||
# --- Bug reports (E16): the Gitea issue tracker a debug session's issue goes to ---
|
||||
ConfigSpec("issues_gitea_url", "str", "issues", "issues_gitea_url"),
|
||||
ConfigSpec("issues_gitea_repo", "str", "issues", "issues_gitea_repo"),
|
||||
ConfigSpec("issues_gitea_token", "str", "issues", "issues_gitea_token", secret=True),
|
||||
# (tmdb_enrich_interval_min is a scheduler job, tuned from the Scheduler dashboard; tmdb_enrich_batch
|
||||
# is a config.py default — both stay out of SPECS, like plex_sync_interval_min.)
|
||||
)
|
||||
@@ -211,17 +235,49 @@ def _default(s: ConfigSpec):
|
||||
return getattr(settings, s.default_attr)
|
||||
|
||||
|
||||
class InvalidSetting(ValueError):
|
||||
"""A value `set_value` refuses, as a reason the UI translates (`errors.refused.setting_<reason>`)
|
||||
plus what the sentence needs — the English message is for logs only. A ValueError still, so a
|
||||
caller that only needs "refused or not" keeps working."""
|
||||
|
||||
def __init__(self, key: str, reason: str, **params: object):
|
||||
super().__init__(f"{key}: {reason} {params}")
|
||||
self.key = key
|
||||
self.reason = reason
|
||||
self.params = params
|
||||
|
||||
|
||||
def _choice(s: ConfigSpec, raw: object) -> str:
|
||||
"""The canonical spelling of `raw` among the key's choices (any letter case), or ValueError."""
|
||||
"""The canonical spelling of `raw` among the key's choices (any letter case), or InvalidSetting."""
|
||||
option = canonical_choice(s.choices, raw)
|
||||
if option is None:
|
||||
raise ValueError(f"{s.key} must be one of: {', '.join(s.choices)}")
|
||||
raise InvalidSetting(s.key, "not_a_choice", choices=", ".join(s.choices))
|
||||
return option
|
||||
|
||||
|
||||
class OutOfRange(ValueError):
|
||||
"""A stored int override that no longer satisfies its spec's bounds."""
|
||||
|
||||
|
||||
def _coerce(s: ConfigSpec, raw: str):
|
||||
if s.type == "int":
|
||||
return int(raw)
|
||||
# Bounds are checked on READ, not only in `set_value`. Bounds get ADDED to a spec
|
||||
# (download_upload_max_bytes gained a `max` when SEC-20 started enforcing the same number at
|
||||
# the edge; its `min` went 0 → 1 during E7) and env defaults get LOWERED, and either leaves
|
||||
# a stored value that was legal when it was written. Serving it is how one layer ends up
|
||||
# acting on a number another layer refuses, with the admin page showing a value outside the
|
||||
# range it renders.
|
||||
#
|
||||
# It raises rather than CLAMPING, and that distinction cost a round: clamping is a silent
|
||||
# rewrite, and silently rewriting is exactly what this module refuses to do elsewhere. The
|
||||
# case that settles it is a stored 0 for download_upload_max_bytes — legal when `min` was 0,
|
||||
# and meaning "no per-file cap" to the route that reads it. Clamped to `min=1` it becomes a
|
||||
# ONE-BYTE cap and uploads stop working with nothing in the log. Raising drops it into the
|
||||
# same fallback the uncoercible path uses: the env default, and one warning.
|
||||
v = int(raw)
|
||||
if (s.min is not None and v < s.min) or (s.max is not None and v > s.max):
|
||||
raise OutOfRange(f"{v} is outside [{s.min}, {s.max}]")
|
||||
return v
|
||||
if s.type == "bool":
|
||||
return raw == "true"
|
||||
if s.type == "choice":
|
||||
@@ -274,14 +330,19 @@ def get(db: Session, key: str):
|
||||
return _coerce(s, raw)
|
||||
except (TypeError, ValueError) as exc:
|
||||
# The admin set this in the UI and it is quietly not in force: the app runs on the env
|
||||
# default while the settings page goes on showing the value that was saved.
|
||||
# default while the settings page goes on showing the value that was saved. An OutOfRange is
|
||||
# the same situation arrived at from the other direction — the value was fine when it was
|
||||
# written and the spec moved under it — so it gets the same treatment and its own code.
|
||||
event = (
|
||||
"sysconfig.override_out_of_range"
|
||||
if isinstance(exc, OutOfRange)
|
||||
else "sysconfig.override_uncoercible"
|
||||
)
|
||||
if (s.key, raw) not in _coerce_warned:
|
||||
_coerce_warned.add((s.key, raw))
|
||||
log.warning(
|
||||
"a stored setting could not be coerced; the default is used instead",
|
||||
extra=logs.event(
|
||||
"sysconfig.override_uncoercible", key=s.key, reason=type(exc).__name__
|
||||
),
|
||||
"a stored setting is not usable; the default is used instead",
|
||||
extra=logs.event(event, key=s.key, reason=type(exc).__name__),
|
||||
)
|
||||
return _default(s)
|
||||
|
||||
@@ -305,11 +366,14 @@ def set_value(db: Session, key: str, value) -> None:
|
||||
(B-core-10). The staged value is visible to a same-session `get()` before the commit."""
|
||||
s = _BY_KEY[key]
|
||||
if s.type == "int":
|
||||
v = int(value)
|
||||
try:
|
||||
v = int(value)
|
||||
except (TypeError, ValueError):
|
||||
raise InvalidSetting(key, "not_a_number") from None
|
||||
if s.min is not None and v < s.min:
|
||||
raise ValueError(f"{key} must be >= {s.min}")
|
||||
raise InvalidSetting(key, "too_low", min=s.min)
|
||||
if s.max is not None and v > s.max:
|
||||
raise ValueError(f"{key} must be <= {s.max}")
|
||||
raise InvalidSetting(key, "too_high", max=s.max)
|
||||
raw = str(v)
|
||||
elif s.type == "bool":
|
||||
raw = "true" if value else "false"
|
||||
@@ -321,7 +385,7 @@ def set_value(db: Session, key: str, value) -> None:
|
||||
stored, encrypted = raw, False
|
||||
if s.secret:
|
||||
if not secrets_manageable():
|
||||
raise RuntimeError("TOKEN_ENCRYPTION_KEY is not configured")
|
||||
raise InvalidSetting(key, "no_encryption_key")
|
||||
stored, encrypted = security.encrypt(raw) or "", True
|
||||
|
||||
row = db.get(SystemConfig, key)
|
||||
|
||||
@@ -43,7 +43,7 @@ from app.downloads.jobdb import (
|
||||
_staging_dir,
|
||||
)
|
||||
from app.downloads.status import STOPPED_BY_ROUTE, JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_stop = threading.Event()
|
||||
@@ -562,7 +562,7 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
|
||||
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
||||
if not job.display_name and asset.title:
|
||||
# display_name is VARCHAR(255); a title can be longer (e.g. a recipe in the title).
|
||||
fields["display_name"] = asset.title[:255]
|
||||
fields["display_name"] = asset.title[:DISPLAY_TEXT_MAX]
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
# Atomic finalize: only marks the job done while it is still running, so a concurrent
|
||||
# cancel/pause is never resurrected (see service.finalize_if_running).
|
||||
|
||||
@@ -9,6 +9,7 @@ uses OAuth.
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Literal
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -108,6 +109,30 @@ def is_quota_refusal(exc: YouTubeError) -> bool:
|
||||
return any(r in _QUOTA_REASONS for r in exc.reason.split(","))
|
||||
|
||||
|
||||
# The token endpoint's answer for a revoked or expired grant — the one refusal a reconnect fixes.
|
||||
_DEAD_GRANT = "invalid_grant"
|
||||
|
||||
# What a refused YouTube action means to the person who asked for it — the three things they can
|
||||
# act on differently: wait for tomorrow's budget, reconnect the account, or try again later.
|
||||
YouTubeRefusal = Literal["quota", "reconnect", "unavailable"]
|
||||
|
||||
|
||||
def refusal_reason(exc: YouTubeError) -> YouTubeRefusal:
|
||||
"""Sort a failure by what the user can do about it. A dead grant (Google refused the refresh
|
||||
with `invalid_grant`, which also drops the grant — see `_drop_dead_grant`) or no usable token at
|
||||
all (raised before any endpoint answered, so with no path and no transport reason) means the
|
||||
account must be reconnected; retrying cannot help. Any other token-endpoint failure — a 5xx, a
|
||||
429, a misconfigured client — leaves the grant as it was: reconnecting would not fix it, and
|
||||
the account would still read as connected, with no connect button to press."""
|
||||
if is_quota_refusal(exc):
|
||||
return "quota"
|
||||
if exc.path == "oauth2/token":
|
||||
return "reconnect" if exc.reason == _DEAD_GRANT else "unavailable"
|
||||
if not exc.path and exc.reason != "transport":
|
||||
return "reconnect"
|
||||
return "unavailable"
|
||||
|
||||
|
||||
def _chunks(items: list, size: int) -> Iterator[list]:
|
||||
for i in range(0, len(items), size):
|
||||
yield items[i : i + size]
|
||||
@@ -181,7 +206,10 @@ class YouTubeClient:
|
||||
},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise _upstream_failure("POST", "oauth2/token", resp)
|
||||
exc = _upstream_failure("POST", "oauth2/token", resp)
|
||||
if exc.reason == _DEAD_GRANT:
|
||||
self._drop_dead_grant(tok)
|
||||
raise exc
|
||||
data = resp.json()
|
||||
access = data["access_token"]
|
||||
tok.access_token = encrypt(access)
|
||||
@@ -191,6 +219,25 @@ class YouTubeClient:
|
||||
log.info("Refreshed access token for user %s", self.user.id)
|
||||
return access
|
||||
|
||||
def _drop_dead_grant(self, tok) -> None:
|
||||
"""Google answered the refresh with `invalid_grant`: the grant was revoked or expired, and no
|
||||
retry will revive it. Forget it, so the account reads as NOT connected — Settings and the
|
||||
onboarding wizard then offer the connect button again (a fresh consent issues a new refresh
|
||||
token). Kept as "granted", the UI showed a live connection with no way to renew it while
|
||||
every call failed. Committed like the refresh above: the caller rolls back on the error."""
|
||||
tok.scopes = None
|
||||
tok.access_token = None
|
||||
tok.refresh_token_enc = None
|
||||
tok.expiry = None
|
||||
self.db.add(tok)
|
||||
self.db.commit()
|
||||
# INFO: the refusal itself is already the warning (`youtube.api.error`, invalid_grant).
|
||||
log.info(
|
||||
"cleared the dead YouTube grant of user %s",
|
||||
self.user.id,
|
||||
extra=logs.event("youtube.grant.cleared"),
|
||||
)
|
||||
|
||||
# --- core request ---
|
||||
def _get(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
// The backend type-check lane (E27 S3) — the gate's ninth lane, and the first thing that has ever
|
||||
// type-checked the ~36k lines of `app/`. Read as JSONC: pyright accepts comments here.
|
||||
//
|
||||
// basedpyright, not mypy: measured on this tree with the dependencies actually installed, both land
|
||||
// ~190-200 errors with the same distribution, so the choice was made on the things that differ —
|
||||
// basedpyright is ~2x faster, needs no MYPYPATH to resolve our `app.*` imports, and has a FIRST-CLASS
|
||||
// baseline (`--writebaseline`), which is the whole ratchet. mypy would have needed a third-party tool
|
||||
// for that.
|
||||
//
|
||||
// It runs in the pytest test image, never on the host: `requirements.lock` cannot be installed on
|
||||
// Windows at all (uvloop refuses to build there), so a host run would type-check against absent
|
||||
// third-party packages and report nonsense. The version is pinned in requirements-dev.txt, which the
|
||||
// image installs — so the lane's version IS the pin, with no host-vs-pin check to drift.
|
||||
//
|
||||
// `pythonVersion` is not free-standing: `/.python-version` is the single source of truth and
|
||||
// `tests/test_python_engine_contract.py` fails this gate if this file, the two Dockerfiles, the CI
|
||||
// container or ruff's target-version disagree with it.
|
||||
//
|
||||
// `include` covers `alembic` for the same reason the ruff lane was widened to it: migrations run on
|
||||
// every container start, so one that raises there crash-loops prod, and "it is generated code" is
|
||||
// not an argument when the generated code is what boots the system. `tests/` is deliberately OUT —
|
||||
// a test's job is to fail loudly when it is wrong, which is the opposite of the silent runtime
|
||||
// failure this lane exists to catch, and annotating fixtures to a checker's satisfaction is churn
|
||||
// the ratchet exists to avoid.
|
||||
{
|
||||
"include": ["app", "alembic"],
|
||||
"extraPaths": ["."],
|
||||
"typeCheckingMode": "basic",
|
||||
"pythonVersion": "3.12",
|
||||
"pythonPlatform": "Linux"
|
||||
}
|
||||
@@ -4,6 +4,10 @@
|
||||
# backend lint lane runs on the host (the gate fails when the host's ruff is not this one). httpx2:
|
||||
# Starlette 1.x's TestClient is built on it (`import httpx2 as httpx`) and only falls back to the
|
||||
# app's own httpx with a deprecation warning on every run.
|
||||
# basedpyright: the backend type-check lane (E27 S3). Pinned like everything else here — a type
|
||||
# checker whose version floats silently changes what "clean" means, and the ratchet baseline it
|
||||
# writes is keyed to the diagnostics THIS version produces.
|
||||
basedpyright==1.40.1
|
||||
pytest==8.4.2
|
||||
ruff==0.15.21
|
||||
httpx2==2.13.0
|
||||
|
||||
@@ -47,7 +47,6 @@ pydantic-settings==2.15.0
|
||||
pydantic==2.13.4
|
||||
pydantic_core==2.46.4
|
||||
python-dotenv==1.2.3
|
||||
python-multipart==0.0.32
|
||||
requests==2.34.2
|
||||
rich==15.0.0
|
||||
starlette==1.6.0
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
fastapi>=0.115,<1.0
|
||||
# Multipart form parsing — required by Starlette/FastAPI to accept file uploads (UploadFile/Form).
|
||||
# The download center's local-file Upload (E7) is the only multipart endpoint in the app.
|
||||
python-multipart>=0.0.9,<1.0
|
||||
uvicorn[standard]>=0.30,<1.0
|
||||
sqlalchemy>=2.0,<2.1
|
||||
psycopg[binary]>=3.2,<4.0
|
||||
|
||||
@@ -7,6 +7,10 @@
|
||||
# the repo wrap at the same width, and it's the width `ruff format` enforces (gated via
|
||||
# `ruff format --check` alongside `ruff check`).
|
||||
line-length = 100
|
||||
# The interpreter this code actually runs on, so ruff judges syntax and upgrade hints against the
|
||||
# right one. Left unset it falls back to a default that has nothing to do with us. `/.python-version`
|
||||
# is the single source of truth; tests/test_python_engine_contract.py holds this to it.
|
||||
target-version = "py312"
|
||||
|
||||
[lint]
|
||||
# Beyond the default set, the rules that hold a LOG LINE to what E22.5 makes of it:
|
||||
|
||||
@@ -15,13 +15,15 @@ tests that COMMIT (the whole point of this lane — transaction-ownership behavi
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy import create_engine, event, text
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
import app.models # noqa: F401 — register every model on Base.metadata
|
||||
import app.state as app_state
|
||||
@@ -100,17 +102,67 @@ def pytest_configure(config):
|
||||
)
|
||||
|
||||
|
||||
def _request_dep(db, unfinished: list[str]):
|
||||
"""`get_db` for a test request: the test's own session, plus the check a shared session hides.
|
||||
|
||||
A route helper flushes and its HANDLER commits (CLAUDE.md). Production's `get_db` closes the
|
||||
session after the request, which rolls back anything flushed but not committed — yet here every
|
||||
request shares the test's session, so a follow-up GET or an assertion reads the flushed write
|
||||
and the suite stays green over a handler that forgot to commit (deep review, debt burn). So a
|
||||
request that ends normally with a write it did not commit — flushed, or still only staged — is
|
||||
recorded, and the test fails on it at teardown. Only the ROOT transaction's end clears it: a
|
||||
savepoint released inside the handler commits nothing. The test's own staged rows are flushed before the request starts — they are the
|
||||
test's business, not the handler's. A request that raises is rolled back, as production's
|
||||
`close()` rolls it back: without that, what a failing handler flushed but never committed stayed
|
||||
readable to the rest of the test — the one state production cannot be in (E16: an issue created
|
||||
on the tracker before a failed upload was kept only by an explicit commit, and nothing saw it
|
||||
missing)."""
|
||||
|
||||
def dep(conn: HTTPConnection):
|
||||
db.flush()
|
||||
state = {"dirty": False}
|
||||
|
||||
def flushed(session, ctx):
|
||||
state["dirty"] = True
|
||||
|
||||
def ended(session, transaction):
|
||||
if transaction.parent is None:
|
||||
state["dirty"] = False
|
||||
|
||||
hooks = (("after_flush", flushed), ("after_transaction_end", ended))
|
||||
for name, fn in hooks:
|
||||
event.listen(db, name, fn)
|
||||
try:
|
||||
yield db
|
||||
staged = db.new or db.deleted or any(db.is_modified(o) for o in db.dirty)
|
||||
if state["dirty"] or staged:
|
||||
unfinished.append(f"{conn.scope.get('method', 'WS')} {conn.url.path}")
|
||||
except Exception:
|
||||
db.rollback()
|
||||
raise
|
||||
finally:
|
||||
for name, fn in hooks:
|
||||
event.remove(db, name, fn)
|
||||
|
||||
return dep
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(_schema):
|
||||
"""A real session against the test Postgres, on a clean (empty) database. The code under test
|
||||
may commit freely."""
|
||||
may commit freely. `db.info["request"]` is the `get_db` override for a request against it — the
|
||||
one every test client uses — and a handler that left a write uncommitted fails the test here."""
|
||||
_truncate_all()
|
||||
session = SessionLocal()
|
||||
unfinished: list[str] = []
|
||||
session.info["request"] = _request_dep(session, unfinished)
|
||||
session.info["unfinished"] = unfinished # the requests recorded so far (the check's own test)
|
||||
try:
|
||||
yield session
|
||||
finally:
|
||||
session.rollback()
|
||||
session.close()
|
||||
assert unfinished == [], "a handler left a write uncommitted"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -168,7 +220,7 @@ def client(db, override_dep, _configured):
|
||||
`current_user` override — so login, the wallet gate and epoch revoke are exercised end to end.
|
||||
Starlette's SessionMiddleware round-trips the signed cookie, so a login in one call authenticates
|
||||
the next on this same instance. No lifespan (the scheduler / HLS sweep stay off)."""
|
||||
override_dep(get_db, lambda: db)
|
||||
override_dep(get_db, db.info["request"])
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@@ -187,7 +239,7 @@ def as_user(db, override_dep, _configured):
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
return registry[uid]
|
||||
|
||||
override_dep(get_db, lambda: db)
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, _resolver)
|
||||
|
||||
def _as(user: User) -> TestClient:
|
||||
@@ -197,6 +249,27 @@ def as_user(db, override_dep, _configured):
|
||||
return _as
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def upload_headers():
|
||||
"""Build the headers `POST /api/downloads/upload` takes since P1.
|
||||
|
||||
Shared, because two test modules drive that route and a second copy of this is exactly the drift
|
||||
that lets the wire shape change in one place only. The body IS the file now; the names ride as
|
||||
percent-encoded headers (`quote` mirrors the client's `encodeURIComponent`, since headers are
|
||||
latin-1 and a Hungarian filename would not survive raw)."""
|
||||
|
||||
def _headers(filename: str, display_name: str | None = None) -> dict[str, str]:
|
||||
headers = {
|
||||
"content-type": "application/octet-stream",
|
||||
"x-upload-filename": quote(filename),
|
||||
}
|
||||
if display_name is not None:
|
||||
headers["x-upload-display-name"] = quote(display_name)
|
||||
return headers
|
||||
|
||||
return _headers
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_rate_limiters():
|
||||
"""Every RateLimiter is a module-level singleton keyed by client IP, and TestClient always
|
||||
|
||||
@@ -100,7 +100,7 @@ def test_the_single_playlist_repull_shares_the_pull_cooldown(db, as_user, make_u
|
||||
db.commit()
|
||||
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: {"ok": True})
|
||||
monkeypatch.setattr(plroutes, "sync_user_playlists", lambda d, u: {"ok": True})
|
||||
monkeypatch.setattr(plroutes, "pull_user_playlists", lambda d, u: {"ok": True})
|
||||
|
||||
c = as_user(user)
|
||||
codes = [c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code for _ in range(4)]
|
||||
|
||||
@@ -168,3 +168,15 @@ def test_deleting_the_user_also_clears_the_way_to_re_invite(db, make_user, as_us
|
||||
assert (
|
||||
db.execute(select(Invite).where(Invite.email == member.email)).scalar_one_or_none() is None
|
||||
)
|
||||
|
||||
|
||||
def test_whitelisting_a_new_address_audits_it_by_id(db, make_user, as_user):
|
||||
"""The audit row was written before the new invite had an id, so it named no target."""
|
||||
from app.models import AuditLog
|
||||
|
||||
admin = make_user(role="admin")
|
||||
r = as_user(admin).post("/api/admin/invites", json={"email": "fresh-e27@example.com"})
|
||||
assert r.status_code == 200
|
||||
inv = db.scalar(select(Invite).where(Invite.email == "fresh-e27@example.com"))
|
||||
row = db.scalar(select(AuditLog).where(AuditLog.target_type == "invite"))
|
||||
assert row is not None and row.target_id == str(inv.id)
|
||||
|
||||
@@ -447,7 +447,7 @@ def test_register_route_creates_the_account_end_to_end(client, db, monkeypatch):
|
||||
|
||||
r = client.post("/auth/register", json={"email": "newbie@example.com", "password": _PW})
|
||||
|
||||
assert r.status_code == 200 and r.json() == {"status": "ok"}
|
||||
assert r.status_code == 200 and r.json() == {"ok": True, "status": "ok"}
|
||||
db.expire_all()
|
||||
user = db.execute(_select(User).where(User.email == "newbie@example.com")).scalar_one_or_none()
|
||||
assert user is not None, "POST /auth/register answered ok but created nothing"
|
||||
|
||||
@@ -0,0 +1,705 @@
|
||||
"""SEC-20 — the global request-body cap (`app/bodylimit.py`).
|
||||
|
||||
Two halves, deliberately kept apart:
|
||||
|
||||
* the MIDDLEWARE tests drive a toy ASGI app, so they pin the mechanism (declared length, counted
|
||||
bytes, the exemption match) with no router, no auth and no database in the way;
|
||||
* the APP tests drive the real application, so they pin that the middleware is actually wired in,
|
||||
sits ahead of the auth dependencies, and refuses in the same shape `capped_body` does.
|
||||
|
||||
The pair that matters most is "an oversized body is refused" together with "a normal body still
|
||||
gets through": the first alone would pass just as well if the cap refused *everything*.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.requests import Request
|
||||
|
||||
from app import bodylimit, debugsession, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import SystemConfig
|
||||
from app.bodylimit import DEFAULT_MAX_BODY_BYTES, BodyLimitMiddleware
|
||||
from app.routes import diagnostics as diagnostics_routes
|
||||
|
||||
SMALL = b"x" * 16
|
||||
OVER = b"x" * (DEFAULT_MAX_BODY_BYTES + 1)
|
||||
|
||||
|
||||
async def _reading_app(scope, receive, send):
|
||||
"""A toy app that reads the whole body the way a route does, then reports its length.
|
||||
|
||||
It uses a real Starlette `Request`, so the faked `http.disconnect` reaches it as the real
|
||||
ClientDisconnect a real handler would see — the middleware's exception path is exercised, not
|
||||
stubbed around.
|
||||
"""
|
||||
body = await Request(scope, receive).body()
|
||||
payload = json.dumps({"read": len(body)}).encode()
|
||||
await send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": 200,
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
}
|
||||
)
|
||||
await send({"type": "http.response.body", "body": payload})
|
||||
|
||||
|
||||
def _client(**kwargs) -> TestClient:
|
||||
return TestClient(BodyLimitMiddleware(_reading_app, **kwargs))
|
||||
|
||||
|
||||
def _chunks(total: int, size: int = 64 * 1024):
|
||||
"""A body with NO Content-Length: httpx sends an iterator chunked, which is exactly the shape a
|
||||
declared-length check cannot see."""
|
||||
sent = 0
|
||||
while sent < total:
|
||||
n = min(size, total - sent)
|
||||
yield b"x" * n
|
||||
sent += n
|
||||
|
||||
|
||||
# --- the mechanism ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_declared_length_over_the_cap_is_refused_without_reading_the_body():
|
||||
r = _client().post("/anything", content=OVER)
|
||||
assert r.status_code == 413
|
||||
assert r.content == bodylimit.TOO_LARGE_BODY
|
||||
|
||||
|
||||
def test_a_body_under_the_cap_is_passed_through_untouched():
|
||||
# The mutation guard for the test above: if the cap refused everything, this would fail.
|
||||
r = _client().post("/anything", content=SMALL)
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"read": len(SMALL)}
|
||||
|
||||
|
||||
def test_a_chunked_body_is_refused_on_the_COUNTED_bytes():
|
||||
# No Content-Length at all, so the only thing that can stop this is counting as it arrives.
|
||||
r = _client().post("/anything", content=_chunks(DEFAULT_MAX_BODY_BYTES + 64 * 1024))
|
||||
assert r.status_code == 413
|
||||
assert r.content == bodylimit.TOO_LARGE_BODY
|
||||
|
||||
|
||||
def test_a_chunked_body_under_the_cap_still_arrives_whole():
|
||||
size = DEFAULT_MAX_BODY_BYTES - 1
|
||||
r = _client().post("/anything", content=_chunks(size))
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"read": size}
|
||||
|
||||
|
||||
def test_an_understated_content_length_does_not_buy_a_pass():
|
||||
# The declared-length check is the cheap half; a client that lies must still be counted out.
|
||||
r = _client().post("/anything", content=OVER, headers={"content-length": "10"})
|
||||
assert r.status_code == 413
|
||||
|
||||
|
||||
def test_exactly_at_the_cap_is_allowed():
|
||||
size = DEFAULT_MAX_BODY_BYTES
|
||||
r = _client(limit=size).post("/anything", content=b"x" * size)
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"read": size}
|
||||
|
||||
|
||||
# --- the per-route ceilings ---------------------------------------------------------------------
|
||||
|
||||
BIG = {("POST", "/api/things/{tid}/blob"): DEFAULT_MAX_BODY_BYTES * 4}
|
||||
|
||||
|
||||
def test_a_route_with_a_raised_ceiling_may_send_a_body_over_the_global_cap():
|
||||
r = _client(routes=BIG).post("/api/things/abc/blob", content=OVER)
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"read": len(OVER)}
|
||||
|
||||
|
||||
def test_a_raised_ceiling_is_still_a_ceiling():
|
||||
"""The difference between this and an exemption, and the reason the list is a mapping: a route
|
||||
that needs more bytes does not thereby get unlimited bytes. Left uncounted, the upload route
|
||||
would let an ANONYMOUS caller stream without bound — FastAPI parses a declared body before the
|
||||
first dependency, so `require_human` has not run yet when the bytes land."""
|
||||
over = b"x" * (BIG[("POST", "/api/things/{tid}/blob")] + 1)
|
||||
r = _client(routes=BIG).post("/api/things/abc/blob", content=over)
|
||||
assert r.status_code == 413
|
||||
assert r.content == bodylimit.TOO_LARGE_BODY
|
||||
|
||||
|
||||
def test_a_raised_ceiling_counts_a_chunked_body_too():
|
||||
cap = BIG[("POST", "/api/things/{tid}/blob")]
|
||||
r = _client(routes=BIG).post("/api/things/abc/blob", content=_chunks(cap + 64 * 1024))
|
||||
assert r.status_code == 413
|
||||
|
||||
|
||||
def test_the_raised_ceiling_is_scoped_to_its_method_and_its_path():
|
||||
client = _client(routes=BIG)
|
||||
# Same path, different method.
|
||||
assert client.put("/api/things/abc/blob", content=OVER).status_code == 413
|
||||
# A neighbour that merely starts the same way.
|
||||
assert client.post("/api/things/abc/blob/extra", content=OVER).status_code == 413
|
||||
|
||||
|
||||
def test_a_path_parameter_cannot_swallow_a_slash():
|
||||
# `{tid}` matching greedily across `/` would hand the raised ceiling to routes without one.
|
||||
mw = BodyLimitMiddleware(_reading_app, routes={("POST", "/a/{tid}/b"): 99})
|
||||
assert mw.limit_for("POST", "/a/one/b") == 99
|
||||
assert mw.limit_for("POST", "/a/one/two/b") == DEFAULT_MAX_BODY_BYTES
|
||||
|
||||
|
||||
def test_every_request_gets_a_finite_ceiling():
|
||||
"""There is no pass-through path. A route that is not listed gets the default; one that is
|
||||
listed gets its own; nothing is simply not counted."""
|
||||
mw = BodyLimitMiddleware(_reading_app)
|
||||
assert mw.limit_for("GET", "/api/anything") == DEFAULT_MAX_BODY_BYTES
|
||||
for (method, template), cap in bodylimit.ROUTE_MAX_BODY_BYTES.items():
|
||||
assert cap > DEFAULT_MAX_BODY_BYTES # the admission rule: raised, never lowered
|
||||
assert mw.limit_for(method, template.replace("{sid}", "abc")) == cap
|
||||
|
||||
|
||||
def _declared_routes(root) -> set[tuple[str, str]]:
|
||||
"""Every (method, path template) the app serves.
|
||||
|
||||
It has to RECURSE: since FastAPI 0.141 / Starlette 1.6 an `include_router` leaves an opaque
|
||||
`_IncludedRouter` in `app.routes` and keeps the real routes under its `original_router`, instead
|
||||
of flattening them into the parent as older versions did. Reading `app.routes` alone finds 35
|
||||
entries and none of ours; walking the tree finds all 252.
|
||||
"""
|
||||
found: set[tuple[str, str]] = set()
|
||||
stack = [root]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
for route in getattr(node, "routes", ()) or ():
|
||||
inner = getattr(route, "original_router", None)
|
||||
if inner is not None:
|
||||
stack.append(inner)
|
||||
continue
|
||||
for method in getattr(route, "methods", None) or ():
|
||||
found.add((method, route.path))
|
||||
return found
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("method", "template"), sorted(bodylimit.ROUTE_MAX_BODY_BYTES))
|
||||
def test_every_listed_template_still_resolves_to_a_real_route(method, template):
|
||||
"""The drift guard. The list names route templates that live in another file; a renamed route
|
||||
must fail the gate here, not lose its allowance silently in production."""
|
||||
from app.main import app
|
||||
|
||||
assert (method, template) in _declared_routes(app)
|
||||
|
||||
|
||||
def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes():
|
||||
"""The headroom the module's docblock claims, held as a RATIO rather than as a figure in prose.
|
||||
The client log sink caps itself at 512 KiB — below the default, which is why it is not listed:
|
||||
the global ceiling never fires on it and raising it could only widen what it accepts.
|
||||
|
||||
`<=` alone would not hold the claim. The docblock calls the 2x margin load-bearing precisely
|
||||
because a ceiling that refuses a legitimate request is a worse bug than the one this module
|
||||
fixes, and a margin of 1x is no margin at all.
|
||||
"""
|
||||
assert diagnostics_routes.BODY_MAX_BYTES * 2 <= DEFAULT_MAX_BODY_BYTES
|
||||
assert not any(
|
||||
path.startswith("/api/diagnostics") for _, path in bodylimit.ROUTE_MAX_BODY_BYTES
|
||||
)
|
||||
|
||||
|
||||
# Every body-reading call site in the routers, as (module, function, limit expression). A route that
|
||||
# reads its own body declares the size it MEANS to accept; if that is above the global default and
|
||||
# its route is not in ROUTE_MAX_BODY_BYTES, this middleware refuses a request the route would have
|
||||
# taken — silently, in production, on a path nobody tested. The point of the list is the third entry.
|
||||
#
|
||||
# BOTH helpers, not just `capped_body`: P1 added `capped_stream_to` as its sibling and the registry
|
||||
# did not follow, so the upload route — the one this whole invariant was written for — was the one
|
||||
# site the drift guard could not see (deep-review round 1). The names come from `_common.py` itself
|
||||
# below, so a THIRD sibling cannot be added without this list noticing either — and a route that
|
||||
# skips the helpers altogether and awaits `request.body()` lands here as `<no ceiling declared>`
|
||||
# rather than vanishing (round 3).
|
||||
#
|
||||
# The module is the path relative to `app/routes/`, so a site under `plex/` is spelled
|
||||
# `plex/admin.py` and cannot collide with the top-level `admin.py`.
|
||||
CAPPED_BODY_SITES = {
|
||||
("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"),
|
||||
("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"),
|
||||
("downloads.py", "spool_upload", "capped_stream_to", "cap"),
|
||||
}
|
||||
|
||||
|
||||
def _body_readers() -> set[str]:
|
||||
"""The names of the body-reading helpers `_common.py` exports — asked of the module rather than
|
||||
listed here, so adding a fourth one cannot quietly leave the registry below behind."""
|
||||
common = pathlib.Path(__file__).resolve().parents[1] / "app" / "routes" / "_common.py"
|
||||
tree = ast.parse(common.read_text("utf-8"), filename=str(common))
|
||||
return {
|
||||
node.name
|
||||
for node in tree.body
|
||||
if isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef))
|
||||
and node.name.startswith("capped_")
|
||||
}
|
||||
|
||||
|
||||
def test_the_registry_covers_every_body_reading_helper():
|
||||
"""The mutation guard for the tests below: if a helper were renamed or added, the site list
|
||||
would still match itself while the new one went uncovered."""
|
||||
assert _body_readers() == {"capped_body", "capped_stream_to"}, (
|
||||
"routes/_common.py grew or renamed a body-reading helper — add its call sites to "
|
||||
"CAPPED_BODY_SITES and this expectation."
|
||||
)
|
||||
|
||||
|
||||
# The ways a route can read its own body WITHOUT one of `_common.py`'s helpers. Every `Request`
|
||||
# parameter under `app/routes/` is named `request`, so the base name is an exact filter and not a
|
||||
# guess — the only other `.json()` in the package is `exc.json()` on a ValidationError.
|
||||
#
|
||||
# Without this set the three guards below know only the `capped_*` naming convention, which means
|
||||
# they are blind to the shape `_common.py` itself is written in. A route spelling `await
|
||||
# request.body()` inline declares no ceiling, hands no connection back, and would be invisible to
|
||||
# all three — reading as "no new sites" rather than as a failure, which is the exact way a drift
|
||||
# guard dies quietly (deep-review round 3).
|
||||
RAW_BODY_READS = {"body", "stream", "form", "json"}
|
||||
|
||||
|
||||
def _call_name(node: ast.Call) -> str | None:
|
||||
if isinstance(node.func, ast.Attribute):
|
||||
return node.func.attr
|
||||
return getattr(node.func, "id", None)
|
||||
|
||||
|
||||
def _body_read_calls(fn: ast.AST, readers: set[str]) -> list[tuple[ast.Call, str]]:
|
||||
"""Every call inside `fn` that reads the request body, as (node, name) in source order."""
|
||||
out: list[tuple[ast.Call, str]] = []
|
||||
for node in ast.walk(fn):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
if (
|
||||
isinstance(node.func, ast.Attribute)
|
||||
and isinstance(node.func.value, ast.Name)
|
||||
and node.func.value.id == "request"
|
||||
and node.func.attr in RAW_BODY_READS
|
||||
):
|
||||
out.append((node, f"request.{node.func.attr}"))
|
||||
elif _call_name(node) in readers:
|
||||
out.append((node, _call_name(node) or ""))
|
||||
return sorted(out, key=lambda pair: (pair[0].lineno, pair[0].col_offset))
|
||||
|
||||
|
||||
def _body_reading_functions() -> dict[tuple[str, str], dict]:
|
||||
"""Every function under `app/routes/` that reads a request body itself, keyed by
|
||||
(module, function).
|
||||
|
||||
Parsed, not grepped. A regex over the call text has to guess how the arguments are spelled, and
|
||||
the shapes it guesses wrong — a bare `4 * 1024 * 1024`, a parameter not named `request` — are
|
||||
the natural ways the NEXT site gets written, so a miss reads as "no new sites" rather than as a
|
||||
failure. `ast` sees every call whatever it looks like.
|
||||
|
||||
Keyed by the ENCLOSING FUNCTION, not just the module: a set keyed by module and limit
|
||||
expression collapses two call sites that spell their limit the same way, which is exactly what
|
||||
a second route copied from the first would do.
|
||||
|
||||
The module half of the key is the path RELATIVE to `app/routes/`, not the basename: the scan
|
||||
recurses, and `admin.py`, `discover.py` and `playlists.py` each exist both at the top level and
|
||||
under `plex/`. Keying by basename would let the second of a colliding pair overwrite the first
|
||||
and take its `releases` flag out of the comparison with it — the same entry-collapse this
|
||||
registry was widened to close.
|
||||
|
||||
`_common.py` is skipped: it is where the helpers live, so its own `request.stream()` loops are
|
||||
the implementation of the rule rather than a violation of it.
|
||||
"""
|
||||
readers = _body_readers()
|
||||
routes_dir = pathlib.Path(__file__).resolve().parents[1] / "app" / "routes"
|
||||
found: dict[tuple[str, str], dict] = {}
|
||||
for py in sorted(routes_dir.rglob("*.py")):
|
||||
if py.name == "_common.py":
|
||||
continue
|
||||
module = py.relative_to(routes_dir).as_posix()
|
||||
tree = ast.parse(py.read_text("utf-8"), filename=str(py))
|
||||
for fn in ast.walk(tree):
|
||||
if not isinstance(fn, (ast.AsyncFunctionDef, ast.FunctionDef)):
|
||||
continue
|
||||
calls = _body_read_calls(fn, readers)
|
||||
if not calls:
|
||||
continue
|
||||
names = [name for _, name in calls]
|
||||
limits = []
|
||||
for node, name in zip([n for n, _ in calls], names):
|
||||
if name.startswith("request."):
|
||||
limits.append("<no ceiling declared>")
|
||||
continue
|
||||
limit = next((kw.value for kw in node.keywords if kw.arg == "limit"), None)
|
||||
if limit is None:
|
||||
pos = 2 if name == "capped_stream_to" else 1
|
||||
limit = node.args[pos] if len(node.args) > pos else None
|
||||
limits.append(ast.unparse(limit) if limit is not None else "<none>")
|
||||
# POSITION, not presence. `any(...)` over the function accepted a release that ran
|
||||
# AFTER the await it exists to precede — the natural spelling for someone reading the
|
||||
# name as "release it when you're done" — while pinning a connection for the whole
|
||||
# transfer. The first body read is what the release has to beat.
|
||||
releases = [
|
||||
node
|
||||
for node in ast.walk(fn)
|
||||
if isinstance(node, ast.Call) and _call_name(node) == "release_read_connection"
|
||||
]
|
||||
first_read = calls[0][0].lineno
|
||||
found[(module, fn.name)] = {
|
||||
"helpers": names,
|
||||
"limits": limits,
|
||||
"releases": any(node.lineno < first_read for node in releases),
|
||||
"releases_late": bool(releases)
|
||||
and not any(node.lineno < first_read for node in releases),
|
||||
}
|
||||
return found
|
||||
|
||||
|
||||
def test_every_body_reading_site_is_known_and_fits_under_its_ceiling():
|
||||
found = {
|
||||
(module, fn, site["helpers"][i], site["limits"][i])
|
||||
for (module, fn), site in _body_reading_functions().items()
|
||||
for i in range(len(site["helpers"]))
|
||||
}
|
||||
assert found == CAPPED_BODY_SITES, (
|
||||
f"a body-reading call site appeared, moved or changed shape: {found ^ CAPPED_BODY_SITES}. "
|
||||
"Each one declares a body size; check it against bodylimit.DEFAULT_MAX_BODY_BYTES and, if "
|
||||
"it is larger, add its route to bodylimit.ROUTE_MAX_BODY_BYTES — then update this list."
|
||||
)
|
||||
# And the known ones really do satisfy the rule.
|
||||
assert diagnostics_routes.BODY_MAX_BYTES <= DEFAULT_MAX_BODY_BYTES
|
||||
assert debugsession.SCREENSHOT_MAX_BYTES > DEFAULT_MAX_BODY_BYTES
|
||||
assert ("POST", "/api/debug-sessions/{sid}/screenshots") in bodylimit.ROUTE_MAX_BODY_BYTES
|
||||
# The upload site's limit is the sysconfig knob, which is bounded BY the edge ceiling rather
|
||||
# than being a constant to compare against — that pairing is its own test below.
|
||||
assert ("POST", "/api/downloads/upload") in bodylimit.ROUTE_MAX_BODY_BYTES
|
||||
|
||||
|
||||
def test_release_read_connection_ends_the_transaction_it_was_called_for(db):
|
||||
"""The helper itself: a SELECT has autobegun, and afterwards it has not."""
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import release_read_connection
|
||||
|
||||
db.execute(select(SystemConfig).limit(1))
|
||||
assert db.in_transaction(), "nothing autobegan, so this test would pass vacuously"
|
||||
release_read_connection(db)
|
||||
assert not db.in_transaction()
|
||||
|
||||
|
||||
def test_release_read_connection_refuses_to_discard_pending_work(db):
|
||||
"""The precondition, ENFORCED rather than described.
|
||||
|
||||
The session belongs to the whole request, not to the dependency calling this — so if something
|
||||
ahead of it ever stages a write, ending the transaction would throw that write away with
|
||||
nothing in the log. `service.enqueue` already learned that a whole-session rollback takes the
|
||||
caller's pending work with it, which is why it uses a SAVEPOINT. A comment cannot stop the next
|
||||
dependency from being added in the wrong place; this can.
|
||||
"""
|
||||
from app.db import release_read_connection
|
||||
|
||||
db.add(SystemConfig(key="probe_pending_write", value="1"))
|
||||
with pytest.raises(RuntimeError, match="pending work"):
|
||||
release_read_connection(db)
|
||||
db.rollback() # the staged row is this test's own
|
||||
|
||||
|
||||
def test_release_read_connection_refuses_to_discard_a_FLUSHED_write(db):
|
||||
"""The half the unit-of-work check cannot see.
|
||||
|
||||
`db.new`/`dirty`/`deleted` empty out the moment a helper flushes, and CLAUDE.md's canonical
|
||||
split is "the handler commits; helpers flush" — so the single most likely future violator is
|
||||
the one shape the obvious check is blind to. Measured before the fix: all three sets empty, the
|
||||
guard silent, the row gone after the rollback.
|
||||
"""
|
||||
from app.db import release_read_connection
|
||||
|
||||
db.add(SystemConfig(key="probe_flushed_write", value="1"))
|
||||
db.flush()
|
||||
assert not (db.new or db.dirty or db.deleted), (
|
||||
"the flush did not clear the unit-of-work, so this test would pass for the wrong reason"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="already WRITTEN"):
|
||||
release_read_connection(db)
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_release_read_connection_refuses_to_discard_a_core_dml_write(db):
|
||||
"""The other half: `db.execute(update(...))` never enters the unit-of-work at all, and
|
||||
`downloads/service.py` writes that way on purpose (a conditional `UPDATE … WHERE status=…` is
|
||||
the honest-states pattern)."""
|
||||
from sqlalchemy import insert
|
||||
|
||||
from app.db import release_read_connection
|
||||
|
||||
db.execute(insert(SystemConfig).values(key="probe_core_dml", value="1"))
|
||||
assert not (db.new or db.dirty or db.deleted), (
|
||||
"Core DML entered the unit-of-work, so this test would pass for the wrong reason"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="already WRITTEN"):
|
||||
release_read_connection(db)
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_every_body_reading_site_hands_its_pooled_connection_back_first():
|
||||
"""Reading the body yourself is what makes a request SLOW inside its own database session.
|
||||
|
||||
The auth (or ownership) dependency that must resolve first is a SELECT, so SQLAlchemy has
|
||||
autobegun and the session is holding one of the pool's connections; awaiting the body then pins
|
||||
it for as long as the client keeps sending — one connection per in-flight upload, `idle in
|
||||
transaction`, with nothing to time it out. `db.release_read_connection` ends that transaction,
|
||||
and every one of these sites needs it.
|
||||
|
||||
Held generically because the three sites arrived one at a time and only the newest was fixed:
|
||||
a fourth body-reading route gets the same requirement automatically.
|
||||
|
||||
BEFORE the read, not merely somewhere in the function. A release that runs after the await
|
||||
satisfies the letter of the rule and none of its purpose — the connection was held for exactly
|
||||
the span the rule exists to protect — so `releases_late` is reported as its own, louder case.
|
||||
"""
|
||||
sites = _body_reading_functions()
|
||||
late = [f"{module}:{fn}" for (module, fn), site in sites.items() if site["releases_late"]]
|
||||
missing = [
|
||||
f"{module}:{fn}"
|
||||
for (module, fn), site in sites.items()
|
||||
if not site["releases"] and not site["releases_late"]
|
||||
]
|
||||
assert not late, (
|
||||
"a route calls db.release_read_connection AFTER it reads the body, which holds the pooled "
|
||||
f"connection for exactly the transfer it was supposed to free: {late}. Move the call above "
|
||||
"the await."
|
||||
)
|
||||
assert not missing, (
|
||||
"a route reads its own request body while still inside its read transaction, pinning a "
|
||||
f"pooled connection for the whole transfer: {missing}. Call "
|
||||
"db.release_read_connection(db) before the await — see routes/downloads.py:spool_upload."
|
||||
)
|
||||
|
||||
|
||||
def test_the_upload_ceiling_and_the_admin_knob_cannot_drift_apart():
|
||||
"""The upload route's in-handler cap is a `sysconfig` value an admin can change; the edge is a
|
||||
constant. If the knob could be set above the edge, an admin raising it would silently start
|
||||
getting 413s from a layer the admin page never mentions.
|
||||
|
||||
Since P1 the two count the SAME bytes — the body is the file, with no multipart envelope
|
||||
wrapped round it — so the old allowance is gone and the edge is the knob's own maximum,
|
||||
exactly. A gap reappearing means someone re-wrapped the upload in an envelope without saying so.
|
||||
|
||||
Equality also means the EDGE is what answers on a default install, where no override has lowered
|
||||
the knob: the declared-length check refuses before a byte is read. That is not a gap to close
|
||||
with headroom (measured: headroom only moves the boundary by its own size, and the route's drain
|
||||
pushes the count back over it) — it is a refusal the client has to be able to read, which is why
|
||||
`errors.tooLarge` exists and why the dialog checks the file against `upload_max_bytes` first.
|
||||
"""
|
||||
spec = next(s for s in sysconfig.SPECS if s.key == "download_upload_max_bytes")
|
||||
edge = bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/downloads/upload")]
|
||||
assert spec.max == edge
|
||||
|
||||
|
||||
def _route_objects(root):
|
||||
"""`_declared_routes`' sibling, keeping the route OBJECT so its FastAPI dependant can be read."""
|
||||
found: dict[tuple[str, str], object] = {}
|
||||
stack = [root]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
for route in getattr(node, "routes", ()) or ():
|
||||
inner = getattr(route, "original_router", None)
|
||||
if inner is not None:
|
||||
stack.append(inner)
|
||||
continue
|
||||
for method in getattr(route, "methods", None) or ():
|
||||
found[(method, route.path)] = route
|
||||
return found
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("method", "template"), sorted(bodylimit.ROUTE_MAX_BODY_BYTES))
|
||||
def test_no_route_with_a_raised_ceiling_declares_a_body_parameter(method, template):
|
||||
"""P1's invariant, and the reason this is a test rather than a comment.
|
||||
|
||||
A raised ceiling is only honest when the route's own cap applies to the bytes AS THEY ARRIVE.
|
||||
FastAPI reads and spools a DECLARED body (`= File(...)`, `= Form(...)`, a Pydantic model)
|
||||
BEFORE the first dependency resolves — so on such a route this middleware's ceiling is the only
|
||||
thing between an anonymous caller and the whole allowance, `require_human` never gets a say, and
|
||||
the route's in-handler cap measures bytes that have already landed. That is exactly what P1
|
||||
closed on the upload route, which sat in `ROUTE_MAX_BODY_BYTES` with a 500 MiB ceiling and a
|
||||
`file: UploadFile = File(...)` body for one release.
|
||||
|
||||
Asked of FastAPI itself rather than parsed or grepped: the router already knows which parameters
|
||||
it will read the body for, and a shape a regex or an AST walk guesses wrong is precisely how the
|
||||
next route slips through.
|
||||
|
||||
`route.body_field`, NOT `route.dependant.body_params`, and the difference is the whole test.
|
||||
`body_params` holds only the parameters of the route's OWN callable; `body_field` is built from
|
||||
`_get_flat_body_params`, which recurses into sub-dependencies — and that is also what FastAPI
|
||||
itself consults when it decides to read the body. Measured on the pinned FastAPI: a body declared
|
||||
inside a sub-dependency leaves `body_params` EMPTY while the body is still read in full before
|
||||
auth, so the first version of this test was green over a reproduction of the very defect it
|
||||
exists to forbid (deep-review round 1). The upload route's own idiom — a body read from a
|
||||
`yield` dependency — is exactly the shape that would have slipped through."""
|
||||
from app.main import app
|
||||
|
||||
route = _route_objects(app).get((method, template))
|
||||
assert route is not None, f"{method} {template} is not a route any more"
|
||||
assert route.body_field is None, (
|
||||
f"{method} {template} declares a body ({route.body_field.name}) on the route or in one of "
|
||||
"its dependencies, so FastAPI reads it before the first dependency runs and the raised "
|
||||
"ceiling is all that stands between an anonymous caller and the whole allowance. Read the "
|
||||
"body yourself, from a dependency declared AFTER the auth one — see "
|
||||
"routes/downloads.py:spool_upload."
|
||||
)
|
||||
|
||||
|
||||
def test_a_stored_override_outside_the_bounds_falls_back_on_READ(db):
|
||||
"""`set_value` refuses an out-of-range int, but neither bound has always been what it is now —
|
||||
the upload knob had no `max` before SEC-20 and its `min` was 0 before E7 — so a value that was
|
||||
legal when it was written can be sitting in the database. Served as-is it is exactly the drift
|
||||
the test above forbids: the route would accept bytes the edge refuses.
|
||||
|
||||
The fallback is the env default and a warning, NOT a clamp. A clamp is a silent rewrite, and on
|
||||
the min side it would be a wrong one: a stored 0 meant "no per-file cap" to the route, and
|
||||
clamped to 1 it becomes a one-byte cap that kills uploads with nothing in the log.
|
||||
"""
|
||||
spec = next(s for s in sysconfig.SPECS if s.key == "download_upload_max_bytes")
|
||||
assert spec.max is not None and spec.min is not None
|
||||
default = settings.download_upload_max_bytes
|
||||
|
||||
row = SystemConfig(key="download_upload_max_bytes", value=str(spec.max * 4))
|
||||
db.add(row)
|
||||
db.commit()
|
||||
assert sysconfig.get_int(db, "download_upload_max_bytes") == default
|
||||
|
||||
row.value = "0" # the legacy "no per-file cap", now below min
|
||||
db.commit()
|
||||
assert sysconfig.get_int(db, "download_upload_max_bytes") == default
|
||||
|
||||
# An override that IS in range must still be honoured — otherwise a bounds predicate written
|
||||
# backwards would make every override inert and the two assertions above could not tell.
|
||||
# Deliberately not `spec.max`: that equals the env default for this key, so the assertion would
|
||||
# be satisfied by the fallback it is meant to rule out.
|
||||
row.value = str(spec.max - 1)
|
||||
db.commit()
|
||||
assert spec.max - 1 != default
|
||||
assert sysconfig.get_int(db, "download_upload_max_bytes") == spec.max - 1
|
||||
|
||||
|
||||
def test_the_cap_sits_inside_the_response_decorators_and_outside_the_setup_gate():
|
||||
"""The registration order, held in code rather than in the comment beside it. `add_middleware`
|
||||
inverts the reading order (last registered = OUTERMOST), which is exactly the kind of thing a
|
||||
prose description gets backwards — it already did once.
|
||||
|
||||
The layers are named, not counted by class: all four `@app.middleware("http")` decorators —
|
||||
including the setup gate, which must be INSIDE — register as the same `BaseHTTPMiddleware`, so a
|
||||
class-shaped assertion passes for every arrangement anyone would plausibly get wrong.
|
||||
|
||||
Outside the cap: only response decorators, so the 413 is dressed like any other answer. Inside
|
||||
it: everything that does work on the caller's behalf — the setup gate opens a DB session per
|
||||
request while an instance is unconfigured, which is the anonymous pre-auth work SEC-20 exists to
|
||||
remove.
|
||||
"""
|
||||
from app.main import app
|
||||
|
||||
def layer(m) -> str:
|
||||
dispatch = m.kwargs.get("dispatch") if m.kwargs else None
|
||||
return dispatch.__name__ if dispatch is not None else getattr(m.cls, "__name__", str(m.cls))
|
||||
|
||||
names = [layer(m) for m in app.user_middleware] # outer → inner
|
||||
i = names.index("BodyLimitMiddleware")
|
||||
assert set(names[:i]) <= {
|
||||
"log_requests",
|
||||
"security_headers",
|
||||
"static_cache_headers",
|
||||
"CORSMiddleware",
|
||||
}, f"something that is not a response decorator sits outside the cap: {names[:i]}"
|
||||
assert "log_requests" in names[:i], "the access log must stay outside, so a 413 is logged"
|
||||
assert "setup_gate" in names[i + 1 :], "the setup gate must not run for an oversized body"
|
||||
assert "SessionMiddleware" in names[i + 1 :]
|
||||
|
||||
|
||||
def test_the_screenshot_ceiling_matches_the_cap_the_route_itself_enforces():
|
||||
assert debugsession.SCREENSHOT_MAX_BYTES > DEFAULT_MAX_BODY_BYTES
|
||||
assert (
|
||||
bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/debug-sessions/{sid}/screenshots")]
|
||||
== debugsession.SCREENSHOT_MAX_BYTES
|
||||
)
|
||||
|
||||
|
||||
# --- the real application -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_global_refusal_carries_the_baseline_security_headers():
|
||||
"""The 413 is a response like any other, so it has to be dressed like one. The middleware sits
|
||||
INSIDE the security-header and CORS layers for exactly this reason: a refusal a browser drops
|
||||
for a missing header is indistinguishable from a network failure, and the user would be shown a
|
||||
generic error instead of the translated `too_large`.
|
||||
|
||||
A bare TestClient on purpose, not the `client` fixture: this is the only test that exercises the
|
||||
middleware ORDER, and that fixture is DB-gated — so on a host without Postgres the one assertion
|
||||
holding the move would have skipped while the file still reported green. The refusal never
|
||||
reaches a database; it is answered above the router.
|
||||
"""
|
||||
from app.main import app
|
||||
|
||||
r = TestClient(app).post(
|
||||
"/auth/password-login", content=OVER, headers={"content-type": "text/plain"}
|
||||
)
|
||||
assert r.status_code == 413
|
||||
assert r.headers.get("x-content-type-options") == "nosniff"
|
||||
assert "referrer-policy" in r.headers
|
||||
|
||||
|
||||
def test_the_login_route_is_refused_before_a_single_dependency_runs(client, monkeypatch):
|
||||
"""`/auth/password-login` is the reason SEC-20 exists: it is anonymous, it declares a body, and
|
||||
FastAPI reads that body in full before the first dependency. `_login_limiter.allow` is the very
|
||||
first statement of the handler, so it is the sharpest available proof that nothing ran."""
|
||||
from app.routes import auth as auth_routes
|
||||
|
||||
reached = False
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
nonlocal reached
|
||||
reached = True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(auth_routes._login_limiter, "allow", _record)
|
||||
|
||||
r = client.post("/auth/password-login", content=OVER, headers={"content-type": "text/plain"})
|
||||
assert r.status_code == 413
|
||||
assert not reached, "the handler ran — the cap is inside the router instead of ahead of it"
|
||||
|
||||
|
||||
def test_a_normal_login_body_still_reaches_the_handler(client, monkeypatch):
|
||||
"""The mutation check for the test above. If the cap refused every body, the 413 test would
|
||||
pass while the app was broken; this one would not."""
|
||||
from app.routes import auth as auth_routes
|
||||
|
||||
reached = False
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
nonlocal reached
|
||||
reached = True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(auth_routes._login_limiter, "allow", _record)
|
||||
|
||||
r = client.post("/auth/password-login", json={"email": "nobody@example.com", "password": "x"})
|
||||
assert r.status_code == 401
|
||||
assert reached
|
||||
|
||||
|
||||
def test_the_global_refusal_is_byte_identical_to_the_per_route_one(db, as_user, make_user):
|
||||
"""Two shapes for one fact is how the `cancelled`/`canceled` bug happened. The log sink caps
|
||||
itself with `capped_body`; its 413 and the middleware's must be the same bytes.
|
||||
|
||||
Signed in on purpose: `current_user` resolves before the body dependency, so an anonymous call
|
||||
would be answered 401 and never reach the per-route cap this compares against. The body is over
|
||||
the SINK's 512 KiB but under the global 1 MiB, so it is `capped_body` that refuses it.
|
||||
"""
|
||||
user = make_user()
|
||||
db.commit()
|
||||
over_the_sink = b"x" * (diagnostics_routes.BODY_MAX_BYTES + 1)
|
||||
assert len(over_the_sink) < DEFAULT_MAX_BODY_BYTES
|
||||
r = as_user(user).post(
|
||||
"/api/diagnostics/events",
|
||||
content=over_the_sink,
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
assert r.status_code == 413
|
||||
# RAW BYTES, not parsed JSON: the two used to differ by the two spaces `json.dumps` puts in by
|
||||
# default and `JSONResponse` does not, and a structural comparison could not see it — which is
|
||||
# the only kind of drift the docblock's promise was ever at risk from.
|
||||
assert r.content == bodylimit.TOO_LARGE_BODY
|
||||
@@ -18,7 +18,7 @@ from app.sync import runner, videos
|
||||
_PUB = datetime(2020, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def test_channel_detail_returns_normal_shorts_live_counts(db, monkeypatch):
|
||||
def test_channel_detail_returns_normal_shorts_live_counts(db, monkeypatch, override_dep):
|
||||
# published_at set so channel_detail skips the YouTube-enrich path (no quota / network in tests).
|
||||
ch = Channel(id="UC_nsl", title="N/S/L Channel", published_at=_PUB)
|
||||
db.add(ch)
|
||||
@@ -40,12 +40,9 @@ def test_channel_detail_returns_normal_shorts_live_counts(db, monkeypatch):
|
||||
|
||||
# The request gate 503s an unconfigured instance; flip the process-level flag (auto-restored).
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: me
|
||||
try:
|
||||
resp = TestClient(app).get(f"/api/channels/{ch.id}")
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: me)
|
||||
resp = TestClient(app).get(f"/api/channels/{ch.id}")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
@@ -56,7 +53,7 @@ def test_channel_detail_returns_normal_shorts_live_counts(db, monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden", [True, False])
|
||||
def test_channel_detail_surfaces_subscription_hidden_flag(db, monkeypatch, hidden):
|
||||
def test_channel_detail_surfaces_subscription_hidden_flag(db, monkeypatch, hidden, override_dep):
|
||||
# S6: the detail GET must expose the per-subscription `hidden` flag so the channel page can offer
|
||||
# the same Unhide toggle the manager has (the flag was previously omitted from the detail dict).
|
||||
ch = Channel(id="UC_hid", title="Hidden?", published_at=_PUB)
|
||||
@@ -68,18 +65,15 @@ def test_channel_detail_surfaces_subscription_hidden_flag(db, monkeypatch, hidde
|
||||
db.commit()
|
||||
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: me
|
||||
try:
|
||||
resp = TestClient(app).get(f"/api/channels/{ch.id}")
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: me)
|
||||
resp = TestClient(app).get(f"/api/channels/{ch.id}")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["hidden"] is hidden
|
||||
|
||||
|
||||
def test_channel_detail_hidden_defaults_false_when_not_subscribed(db, monkeypatch):
|
||||
def test_channel_detail_hidden_defaults_false_when_not_subscribed(db, monkeypatch, override_dep):
|
||||
# No subscription row → the flag can't be True; it must serialize as False, not raise.
|
||||
ch = Channel(id="UC_hid_ns", title="Unsub", published_at=_PUB)
|
||||
me = User(email="hidns@example.com")
|
||||
@@ -87,12 +81,9 @@ def test_channel_detail_hidden_defaults_false_when_not_subscribed(db, monkeypatc
|
||||
db.commit()
|
||||
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: me
|
||||
try:
|
||||
resp = TestClient(app).get(f"/api/channels/{ch.id}")
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: me)
|
||||
resp = TestClient(app).get(f"/api/channels/{ch.id}")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["hidden"] is False
|
||||
@@ -115,33 +106,30 @@ def _setup_channel_and_sub(db, *, email, is_demo=False):
|
||||
return ch, user
|
||||
|
||||
|
||||
def _patch_deep_requested(db, monkeypatch, user):
|
||||
def _patch_deep_requested(db, monkeypatch, override_dep, user):
|
||||
calls: list[tuple] = []
|
||||
# background.add_task runs after the response in TestClient; record what would be scheduled.
|
||||
monkeypatch.setattr(channels_route, "backfill_channel_now", lambda *a, **k: calls.append(a))
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
resp = TestClient(app).patch("/api/channels/UC_wire", json={"deep_requested": True})
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: user)
|
||||
resp = TestClient(app).patch("/api/channels/UC_wire", json={"deep_requested": True})
|
||||
return resp, calls
|
||||
|
||||
|
||||
def test_deep_request_schedules_immediate_backfill_for_non_demo(db, monkeypatch):
|
||||
def test_deep_request_schedules_immediate_backfill_for_non_demo(db, monkeypatch, override_dep):
|
||||
ch, me = _setup_channel_and_sub(db, email="wire@example.com")
|
||||
resp, calls = _patch_deep_requested(db, monkeypatch, me)
|
||||
resp, calls = _patch_deep_requested(db, monkeypatch, override_dep, me)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["deep_requested"] is True
|
||||
# scheduled exactly once, with (channel_id, user_id) — the args backfill_channel_now needs.
|
||||
assert calls == [(ch.id, me.id)]
|
||||
|
||||
|
||||
def test_deep_request_does_not_schedule_immediate_backfill_for_demo(db, monkeypatch):
|
||||
def test_deep_request_does_not_schedule_immediate_backfill_for_demo(db, monkeypatch, override_dep):
|
||||
# The SHARED demo account must not be able to kick immediate full backfills (budget drain).
|
||||
_ch, demo = _setup_channel_and_sub(db, email="demo@example.com", is_demo=True)
|
||||
resp, calls = _patch_deep_requested(db, monkeypatch, demo)
|
||||
resp, calls = _patch_deep_requested(db, monkeypatch, override_dep, demo)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert calls == [], "demo must not schedule an immediate backfill"
|
||||
# …and deep_requested is not persisted on the shared demo subscription either.
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
"""A channel's metadata is fetched again once it is stale, not only the first time it is seen.
|
||||
|
||||
Banners, avatars, descriptions and counts change on YouTube, and an old banner URL stops resolving:
|
||||
a channel that kept the metadata of the day it was first imported showed a dead banner forever. The
|
||||
subscription import and the channel page both refresh what is older than `DETAILS_MAX_AGE`, and
|
||||
leave fresh metadata alone (each refresh spends quota). DB lane.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models import Channel, Subscription
|
||||
from app.routes import channels
|
||||
from app.sync import subscriptions
|
||||
from app.sync.subscriptions import DETAILS_MAX_AGE, import_subscriptions
|
||||
|
||||
NOW = datetime.now(timezone.utc)
|
||||
FRESH = NOW - timedelta(days=1)
|
||||
STALE = NOW - DETAILS_MAX_AGE - timedelta(hours=1)
|
||||
|
||||
|
||||
def _item(cid: str) -> dict:
|
||||
return {
|
||||
"id": cid,
|
||||
"snippet": {"title": f"{cid} renamed", "publishedAt": "2010-01-01T00:00:00Z"},
|
||||
"brandingSettings": {"image": {"bannerExternalUrl": f"https://yt3.example/{cid}-new"}},
|
||||
}
|
||||
|
||||
|
||||
def _youtube(asked: list[list[str]], subscribed: list[str], gone: tuple[str, ...] = ()):
|
||||
class _Client:
|
||||
def __init__(self, db, user):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def iter_subscriptions(self):
|
||||
for cid in subscribed:
|
||||
yield {"channel_id": cid, "title": cid, "yt_subscription_id": f"sub-{cid}"}
|
||||
|
||||
def get_channels(self, ids):
|
||||
asked.append(sorted(ids))
|
||||
# A terminated / deleted channel is simply absent from `channels.list`'s answer.
|
||||
return [_item(cid) for cid in ids if cid not in gone]
|
||||
|
||||
return _Client
|
||||
|
||||
|
||||
def _channel(db, cid: str, synced: datetime | None) -> None:
|
||||
db.add(
|
||||
Channel(
|
||||
id=cid,
|
||||
title=cid,
|
||||
banner_url=f"https://yt3.example/{cid}-old",
|
||||
published_at=datetime(2010, 1, 1, tzinfo=timezone.utc),
|
||||
details_synced_at=synced,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_the_import_refreshes_stale_and_missing_details_and_leaves_fresh_ones(
|
||||
db, make_user, monkeypatch
|
||||
):
|
||||
user = make_user()
|
||||
_channel(db, "UCfresh", FRESH)
|
||||
_channel(db, "UCstale", STALE)
|
||||
_channel(db, "UCnever", None)
|
||||
db.flush()
|
||||
for cid in ("UCfresh", "UCstale", "UCnever"):
|
||||
db.add(Subscription(user_id=user.id, channel_id=cid))
|
||||
db.commit()
|
||||
asked: list[list[str]] = []
|
||||
monkeypatch.setattr(
|
||||
subscriptions, "YouTubeClient", _youtube(asked, ["UCfresh", "UCstale", "UCnever"])
|
||||
)
|
||||
|
||||
result = import_subscriptions(db, user)
|
||||
|
||||
assert asked == [["UCnever", "UCstale"]]
|
||||
assert result["channels_detailed"] == 2
|
||||
stale = db.get(Channel, "UCstale")
|
||||
assert stale is not None
|
||||
assert stale.banner_url == "https://yt3.example/UCstale-new"
|
||||
assert stale.details_synced_at is not None and stale.details_synced_at > STALE
|
||||
fresh = db.get(Channel, "UCfresh")
|
||||
assert fresh is not None and fresh.banner_url == "https://yt3.example/UCfresh-old"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("synced", "refreshed"), [(STALE, True), (FRESH, False)])
|
||||
def test_the_channel_page_refreshes_a_stale_channel_only(
|
||||
db, make_user, monkeypatch, synced, refreshed
|
||||
):
|
||||
user = make_user()
|
||||
_channel(db, "UCpage", synced)
|
||||
db.commit()
|
||||
asked: list[list[str]] = []
|
||||
monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, []))
|
||||
|
||||
body = channels.channel_detail("UCpage", user=user, db=db)
|
||||
|
||||
assert asked == ([["UCpage"]] if refreshed else [])
|
||||
suffix = "new" if refreshed else "old"
|
||||
assert body["banner_url"] == f"https://yt3.example/UCpage-{suffix}"
|
||||
|
||||
|
||||
def test_a_channel_youtube_no_longer_returns_is_not_asked_for_on_every_view(
|
||||
db, make_user, monkeypatch
|
||||
):
|
||||
"""A terminated channel is missing from `channels.list`'s answer, so nothing was stamped and the
|
||||
channel page asked for it again (1 quota unit) on every single view. The miss is stamped like a
|
||||
fetch, so the next ask waits for `DETAILS_MAX_AGE` like any other channel."""
|
||||
user = make_user()
|
||||
db.add(Channel(id="UCgone", title="UCgone"))
|
||||
db.commit()
|
||||
asked: list[list[str]] = []
|
||||
monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, [], gone=("UCgone",)))
|
||||
|
||||
channels.channel_detail("UCgone", user=user, db=db)
|
||||
channels.channel_detail("UCgone", user=user, db=db)
|
||||
|
||||
assert asked == [["UCgone"]]
|
||||
|
||||
|
||||
def test_the_import_stamps_a_channel_youtube_no_longer_returns(db, make_user, monkeypatch):
|
||||
user = make_user()
|
||||
_channel(db, "UCgone", STALE)
|
||||
db.flush()
|
||||
db.add(Subscription(user_id=user.id, channel_id="UCgone"))
|
||||
db.commit()
|
||||
asked: list[list[str]] = []
|
||||
monkeypatch.setattr(
|
||||
subscriptions, "YouTubeClient", _youtube(asked, ["UCgone"], gone=("UCgone",))
|
||||
)
|
||||
|
||||
import_subscriptions(db, user)
|
||||
import_subscriptions(db, user)
|
||||
|
||||
assert asked == [["UCgone"]]
|
||||
@@ -17,7 +17,7 @@ from app.main import app
|
||||
from app.models import Channel, ChannelTag, Tag, User
|
||||
|
||||
|
||||
def test_channel_detail_returns_only_this_users_tag_ids(db, monkeypatch):
|
||||
def test_channel_detail_returns_only_this_users_tag_ids(db, monkeypatch, override_dep):
|
||||
# published_at set so channel_detail skips the YouTube-enrich path (no quota / network in tests).
|
||||
ch = Channel(
|
||||
id="UC_tags_test",
|
||||
@@ -43,13 +43,10 @@ def test_channel_detail_returns_only_this_users_tag_ids(db, monkeypatch):
|
||||
|
||||
# The request gate 503s an unconfigured instance; flip the process-level flag (auto-restored).
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: me
|
||||
try:
|
||||
client = TestClient(app) # no context-manager: skip lifespan (scheduler/HLS sweep)
|
||||
resp = client.get(f"/api/channels/{ch.id}")
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: me)
|
||||
client = TestClient(app) # no context-manager: skip lifespan (scheduler/HLS sweep)
|
||||
resp = client.get(f"/api/channels/{ch.id}")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["tag_ids"] == [mine.id]
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -68,6 +68,7 @@ def test_raising_the_level_sets_the_servers_expiry_whatever_the_client_sends(db,
|
||||
"level": "INFO",
|
||||
"until": None,
|
||||
"retention_days": 7,
|
||||
"session_minutes": 60,
|
||||
}
|
||||
|
||||
|
||||
@@ -82,6 +83,7 @@ def test_a_lapsed_level_reads_as_the_base_in_me(db, client, make_user):
|
||||
"level": "INFO",
|
||||
"until": None,
|
||||
"retention_days": 7,
|
||||
"session_minutes": 60,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -544,6 +544,73 @@ def test_shelf_resolves_a_playlist_to_its_first_unwatched_item(db):
|
||||
assert card["next"]["id"] == "vmed"
|
||||
|
||||
|
||||
def _count_statements(db, fn) -> int:
|
||||
"""How many SQL statements `fn` sends through the session's connection."""
|
||||
from sqlalchemy import event
|
||||
|
||||
seen: list[str] = []
|
||||
engine = db.get_bind()
|
||||
listener = lambda *a, **k: seen.append(a[2]) # noqa: E731 — (conn, cursor, statement, …)
|
||||
event.listen(engine, "before_cursor_execute", listener)
|
||||
try:
|
||||
fn()
|
||||
finally:
|
||||
event.remove(engine, "before_cursor_execute", listener)
|
||||
return len(seen)
|
||||
|
||||
|
||||
def test_the_shelf_resolves_each_channel_and_playlist_on_its_own_terms(db):
|
||||
"""Batched, the per-channel count and newest video come from one windowed query: each channel
|
||||
must still get ITS count and ITS newest, not the shelf's overall first row."""
|
||||
user, ch = _seed_yt(db)
|
||||
other = Channel(id="UCother", title="Other")
|
||||
db.add(other)
|
||||
db.flush()
|
||||
db.add(Video(id="vother", channel_id=other.id, title="x", published_at=NOW - timedelta(days=9)))
|
||||
pl = Playlist(user_id=user.id, name="Két elem")
|
||||
empty = Playlist(user_id=user.id, name="Üres")
|
||||
db.add_all([pl, empty])
|
||||
db.flush()
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id="vlong", position=0))
|
||||
for i, target in enumerate(
|
||||
[
|
||||
{"channel_id": ch.id},
|
||||
{"channel_id": other.id},
|
||||
{"playlist_id": pl.id},
|
||||
{"playlist_id": empty.id},
|
||||
]
|
||||
):
|
||||
db.add(ComfortPin(user_id=user.id, position=i, **target))
|
||||
db.commit()
|
||||
|
||||
got = [
|
||||
(c.get("unwatched_count", c.get("item_count")), (c["next"] or {}).get("id"))
|
||||
for c in pins.shelf(db, user)
|
||||
]
|
||||
assert got == [(3, "vshort"), (1, "vother"), (1, "vlong"), (0, None)]
|
||||
|
||||
|
||||
def test_the_shelf_costs_the_same_queries_for_one_pin_or_many(db):
|
||||
"""A channel pin used to cost three queries and a playlist pin two, each; the shelf now
|
||||
resolves every channel and every playlist together."""
|
||||
user, ch = _seed_yt(db)
|
||||
extra = [Channel(id=f"UCx{i}", title=f"x{i}") for i in range(3)]
|
||||
lists = [Playlist(user_id=user.id, name=f"p{i}") for i in range(4)]
|
||||
db.add_all(extra + lists)
|
||||
db.flush()
|
||||
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=0))
|
||||
db.add(ComfortPin(user_id=user.id, playlist_id=lists[0].id, position=1))
|
||||
db.commit()
|
||||
few = _count_statements(db, lambda: pins.shelf(db, user))
|
||||
|
||||
for i, c in enumerate(extra):
|
||||
db.add(ComfortPin(user_id=user.id, channel_id=c.id, position=10 + i))
|
||||
for i, p in enumerate(lists[1:]):
|
||||
db.add(ComfortPin(user_id=user.id, playlist_id=p.id, position=20 + i))
|
||||
db.commit()
|
||||
assert _count_statements(db, lambda: pins.shelf(db, user)) == few
|
||||
|
||||
|
||||
def test_a_pinned_video_survives_leaving_the_feed_scope(db):
|
||||
"""The shelf is the fallback surface. An entry silently vanishing because the user unsubscribed
|
||||
from the channel is exactly the failure it must not have."""
|
||||
@@ -940,3 +1007,115 @@ def test_deleting_a_pin_reports_what_went(db, as_user):
|
||||
r = c.delete(f"/api/discover/pins/{pin_id}")
|
||||
assert r.status_code == 200 and r.json() == {"deleted": pin_id}
|
||||
assert c.get("/api/discover/pins").json()["items"] == []
|
||||
|
||||
|
||||
def test_with_plex_off_the_home_shelf_hides_plex_pins_but_keeps_them(db, as_user):
|
||||
"""The owner's call (2026-09-19): with the deployment's Plex toggle off, the Home shelf goes
|
||||
YouTube-only like the rails beside it — a Plex tile there offered a Play that landed on a gated
|
||||
module. The hero's comfort tier draws from the same filtered shelf, and the pin menus' own
|
||||
listing still sees every pin, so nothing the user curated is lost."""
|
||||
user, _ = _seed_yt(db)
|
||||
_seed_plex(db, user)
|
||||
db.add_all(
|
||||
[
|
||||
ComfortPin(
|
||||
user_id=user.id,
|
||||
plex_item_id=db.scalar(select(PlexItem.id).where(PlexItem.rating_key == "pm1")),
|
||||
position=0,
|
||||
),
|
||||
ComfortPin(user_id=user.id, video_id="vlong", position=1),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
c = as_user(user)
|
||||
assert [t["id"] for t in c.get("/api/discover").json()["comfort"]] == ["vlong"]
|
||||
assert c.get("/api/discover/pick?start=1").json()["hero"]["card"]["id"] == "vlong"
|
||||
assert {t["id"] for t in c.get("/api/discover/pins").json()["items"]} == {"pm1", "vlong"}
|
||||
_enable_plex(db)
|
||||
assert [t["id"] for t in c.get("/api/discover").json()["comfort"]] == ["pm1", "vlong"]
|
||||
|
||||
|
||||
def test_a_reorder_of_the_visible_pins_keeps_the_hidden_ones_in_place(db, as_user):
|
||||
"""The client sends the order it SHOWS. With Plex off that leaves the Plex pins out, and
|
||||
numbering only the listed ids 0..n-1 put two pins on one position and moved the hidden one about
|
||||
when Plex came back. The listed pins trade the slots they hold; the rest stay put."""
|
||||
user, ch = _seed_yt(db)
|
||||
_seed_plex(db, user)
|
||||
c = as_user(user)
|
||||
_enable_plex(db)
|
||||
a = c.post("/api/discover/pins", json={"kind": "video", "ref": "vmed"}).json()["id"]
|
||||
hidden = c.post("/api/discover/pins", json={"kind": "plex_item", "ref": "pm1"}).json()["id"]
|
||||
b = c.post("/api/discover/pins", json={"kind": "channel", "ref": ch.id}).json()["id"]
|
||||
assert c.post("/api/discover/pins/reorder", json={"ids": [b, b, a]}).status_code == 200
|
||||
assert [p["pin_id"] for p in c.get("/api/discover/pins").json()["items"]] == [b, hidden, a]
|
||||
positions = db.scalars(select(ComfortPin.position).where(ComfortPin.user_id == user.id)).all()
|
||||
assert sorted(positions) == [0, 1, 2]
|
||||
|
||||
|
||||
def test_a_hero_from_a_show_pin_names_the_pin_it_came_from(db, as_user):
|
||||
"""A show's next EPISODE is not itself pinned, so the hero's menu used to look the episode up,
|
||||
miss, and offer "Pin" under "From your shelf" — creating a second pin. The card now carries
|
||||
the pin it was resolved from, which the menu acts on (owner's call: "Unpin")."""
|
||||
user, _ = _seed_yt(db)
|
||||
_seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
c = as_user(user)
|
||||
assert c.post("/api/discover/pins", json={"kind": "plex_show", "ref": "ps1"}).status_code == 200
|
||||
got = c.get("/api/discover/pick?start=1").json()["hero"]
|
||||
assert got["tier"] == "comfort" and got["card"]["id"] == "ps1e1"
|
||||
assert got["card"]["pinned_from"] == {
|
||||
"source": "plex",
|
||||
"kind": "show",
|
||||
"id": "ps1",
|
||||
"title": "A Show",
|
||||
}
|
||||
|
||||
|
||||
def test_two_overlapping_reorders_never_share_a_slot(db, as_user):
|
||||
"""`reorder` writes only the rows that change. Unlocked, two tabs reordering from the same
|
||||
snapshot each skipped the rows that already matched in THEIR snapshot and interleaved row by
|
||||
row — two pins on one slot (deep review, debt burn). Locked, the second waits for the first and
|
||||
deals from its result: here a partial list (a tab with Plex off, one pin hidden), whose left-out
|
||||
pin must stay where the FIRST reorder put it — a lock taken in position order handed the waiter
|
||||
fresh rows in its stale order, and it dealt that pin back to its old slot (round 2)."""
|
||||
import threading
|
||||
|
||||
from app.db import SessionLocal
|
||||
|
||||
user, ch = _seed_yt(db)
|
||||
c = as_user(user)
|
||||
p1 = c.post("/api/discover/pins", json={"kind": "video", "ref": "vmed"}).json()["id"]
|
||||
p2 = c.post("/api/discover/pins", json={"kind": "channel", "ref": ch.id}).json()["id"]
|
||||
p3 = c.post("/api/discover/pins", json={"kind": "video", "ref": "vlong"}).json()["id"]
|
||||
uid = user.id
|
||||
|
||||
with SessionLocal() as tab_a, SessionLocal() as tab_b, SessionLocal() as probe:
|
||||
pins.reorder(tab_a, tab_a.get(User, uid), [p2, p3, p1]) # written, not yet committed
|
||||
errors: list[BaseException] = []
|
||||
|
||||
def second_tab():
|
||||
try:
|
||||
pins.reorder(tab_b, tab_b.get(User, uid), [p3, p2])
|
||||
tab_b.commit()
|
||||
except BaseException as exc: # noqa: BLE001 — re-raised in the test thread below
|
||||
errors.append(exc)
|
||||
|
||||
worker = threading.Thread(target=second_tab)
|
||||
worker.start()
|
||||
# Commit A only once B is blocked on a row lock, so B's read is the stale snapshot.
|
||||
waiting = 0
|
||||
for _ in range(200):
|
||||
waiting = probe.scalar(text("SELECT count(*) FROM pg_locks WHERE NOT granted"))
|
||||
probe.rollback()
|
||||
if waiting:
|
||||
break
|
||||
worker.join(0.05)
|
||||
assert waiting, "tab B never blocked — the race was not exercised"
|
||||
tab_a.commit()
|
||||
worker.join(10)
|
||||
assert not worker.is_alive() and errors == []
|
||||
|
||||
db.expire_all()
|
||||
positions = db.scalars(select(ComfortPin.position).where(ComfortPin.user_id == uid)).all()
|
||||
assert sorted(positions) == [0, 1, 2]
|
||||
assert [p["pin_id"] for p in c.get("/api/discover/pins").json()["items"]] == [p3, p2, p1]
|
||||
|
||||
@@ -15,34 +15,31 @@ from app.main import app
|
||||
from app.models import User
|
||||
|
||||
|
||||
def _get(db, user, monkeypatch, path):
|
||||
def _get(db, user, monkeypatch, override_dep, path):
|
||||
monkeypatch.setattr(
|
||||
state, "_configured_cache", True
|
||||
) # the request gate 503s an unconfigured app
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
client = TestClient(app) # no context manager: skip lifespan (scheduler/HLS sweep)
|
||||
return client.get(path)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: user)
|
||||
client = TestClient(app) # no context manager: skip lifespan (scheduler/HLS sweep)
|
||||
return client.get(path)
|
||||
|
||||
|
||||
def test_malformed_feed_cursor_returns_400(db, monkeypatch):
|
||||
def test_malformed_feed_cursor_returns_400(db, monkeypatch, override_dep):
|
||||
user = User(email="feedcursor@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
resp = _get(db, user, monkeypatch, "/api/feed?cursor=not-a-real-cursor")
|
||||
resp = _get(db, user, monkeypatch, override_dep, "/api/feed?cursor=not-a-real-cursor")
|
||||
assert resp.status_code == 400
|
||||
assert resp.json()["detail"] == "Invalid feed cursor"
|
||||
assert resp.json()["detail"] == {"code": "refused", "reason": "invalid_cursor"}
|
||||
|
||||
|
||||
def test_feed_without_cursor_is_200(db, monkeypatch):
|
||||
def test_feed_without_cursor_is_200(db, monkeypatch, override_dep):
|
||||
# Sanity: the same endpoint without a cursor is a normal 200 (proves the 400 above is the
|
||||
# cursor path, not a globally-broken request). A fresh user has no subscriptions → empty feed.
|
||||
user = User(email="feednocursor@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
resp = _get(db, user, monkeypatch, "/api/feed")
|
||||
resp = _get(db, user, monkeypatch, override_dep, "/api/feed")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["items"] == []
|
||||
|
||||
@@ -116,10 +116,10 @@ def test_the_periodic_pass_still_reaps_what_nobody_has_touched_for_the_idle_limi
|
||||
|
||||
def test_touching_a_session_keeps_it_through_the_short_rule(tmp_path, sessions):
|
||||
s = _Session(tmp_path, "paused", finished=True, idle_s=90)
|
||||
sessions["12345"] = s
|
||||
stream.touch_session("12345")
|
||||
sessions[(1, "12345")] = s
|
||||
stream.touch_session(1, "12345")
|
||||
assert stream.reap_idle() == 0
|
||||
assert "12345" in sessions and s.dir.exists() and not s.proc.killed
|
||||
assert (1, "12345") in sessions and s.dir.exists() and not s.proc.killed
|
||||
|
||||
|
||||
# --- the two call sites, driven through the real routes (DB lane) -------------------------------
|
||||
@@ -142,13 +142,15 @@ def _library_item(db, rating_key: str, *, playable: str = "remux"):
|
||||
return it
|
||||
|
||||
|
||||
def _client(db, user):
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
def _client(db, user, override_dep):
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: user)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_a_paused_viewers_checkpoint_keeps_their_session_alive(db, monkeypatch, tmp_path, sessions):
|
||||
def test_a_paused_viewers_checkpoint_keeps_their_session_alive(
|
||||
db, monkeypatch, tmp_path, sessions, override_dep
|
||||
):
|
||||
"""The whole reason the short rule is safe again: a paused player fetches nothing, so
|
||||
`last_access` never moved on its own — but it does keep checkpointing its position."""
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
@@ -157,21 +159,18 @@ def test_a_paused_viewers_checkpoint_keeps_their_session_alive(db, monkeypatch,
|
||||
db.add(user)
|
||||
db.commit()
|
||||
s = _Session(tmp_path, "paused", finished=True, idle_s=90)
|
||||
sessions["paused1"] = s
|
||||
try:
|
||||
resp = _client(db, user).post(
|
||||
f"/api/plex/item/{it.rating_key}/progress",
|
||||
json={"position_seconds": 600, "duration_seconds": 3000, "final": False},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
sessions[(user.id, "paused1")] = s
|
||||
resp = _client(db, user, override_dep).post(
|
||||
f"/api/plex/item/{it.rating_key}/progress",
|
||||
json={"position_seconds": 600, "duration_seconds": 3000, "final": False},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert stream.reap_idle() == 0, "the checkpoint did not say the viewer is still there"
|
||||
assert "paused1" in sessions and not s.proc.killed
|
||||
assert (user.id, "paused1") in sessions and not s.proc.killed
|
||||
|
||||
|
||||
def test_starting_a_session_leaves_another_viewers_paused_one_alone(
|
||||
db, monkeypatch, tmp_path, sessions
|
||||
db, monkeypatch, tmp_path, sessions, override_dep
|
||||
):
|
||||
"""The opportunistic sweep this route used to run was not scoped to the key being started: it
|
||||
walked the WHOLE registry with the 30 s finished rule, so pressing play on any item killed a
|
||||
@@ -183,9 +182,9 @@ def test_starting_a_session_leaves_another_viewers_paused_one_alone(
|
||||
db.add(user)
|
||||
db.commit()
|
||||
paused = _Session(tmp_path, "paused", finished=True, idle_s=90)
|
||||
sessions["someone-else"] = paused
|
||||
sessions[(999, "someone-else")] = paused
|
||||
|
||||
def fake_start(db_, item, start, audio=None, aoff=0.0, multi=False, transcode=False):
|
||||
def fake_start(db_, item, user_id, start, audio=None, aoff=0.0, multi=False, transcode=False):
|
||||
return SimpleNamespace(
|
||||
id="sid",
|
||||
media_start_s=start,
|
||||
@@ -196,9 +195,6 @@ def test_starting_a_session_leaves_another_viewers_paused_one_alone(
|
||||
)
|
||||
|
||||
monkeypatch.setattr(stream, "start_session", fake_start)
|
||||
try:
|
||||
resp = _client(db, user).post(f"/api/plex/stream/{started.rating_key}/session")
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
resp = _client(db, user, override_dep).post(f"/api/plex/stream/{started.rating_key}/session")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert "someone-else" in sessions and not paused.proc.killed
|
||||
assert (999, "someone-else") in sessions and not paused.proc.killed
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""No refusal answers in a hand-written English sentence.
|
||||
|
||||
The UI is bilingual and translates a structured `detail` (`{code, reason, …}`), but a string detail it
|
||||
can only print as it came — so ~150 of them reached the Hungarian UI verbatim, in the blocking error
|
||||
dialog and on every inline surface that shows `HttpError.detail`. `app/refusal.py` is the family they
|
||||
became; this test is what keeps a new one from being written the old way. Pure: it reads the sources.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import pathlib
|
||||
|
||||
APP = pathlib.Path(__file__).resolve().parents[1] / "app"
|
||||
|
||||
|
||||
def _is_text(node: ast.AST) -> bool:
|
||||
"""A detail that is a sentence: a literal, an f-string, `str(exc)`, or `"…" + x`."""
|
||||
if isinstance(node, ast.Constant) and isinstance(node.value, str):
|
||||
return True
|
||||
if isinstance(node, ast.JoinedStr):
|
||||
return True
|
||||
if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "str":
|
||||
return True
|
||||
if isinstance(node, ast.BoolOp):
|
||||
return any(_is_text(v) for v in node.values)
|
||||
if isinstance(node, ast.BinOp):
|
||||
return _is_text(node.left) or _is_text(node.right)
|
||||
return False
|
||||
|
||||
|
||||
def test_no_http_exception_carries_a_sentence():
|
||||
offenders = []
|
||||
for path in sorted(APP.rglob("*.py")):
|
||||
for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))):
|
||||
if not (
|
||||
isinstance(node, ast.Call)
|
||||
and getattr(node.func, "id", getattr(node.func, "attr", "")) == "HTTPException"
|
||||
):
|
||||
continue
|
||||
kw = {k.arg: k.value for k in node.keywords}
|
||||
detail = kw.get("detail", node.args[1] if len(node.args) > 1 else None)
|
||||
if detail is not None and _is_text(detail):
|
||||
offenders.append(f"{path.relative_to(APP).as_posix()}:{node.lineno}")
|
||||
assert offenders == [], (
|
||||
"a refusal with an English sentence for a detail — raise `refused(status, reason)` "
|
||||
f"(app/refusal.py) and name the reason in both locales: {offenders}"
|
||||
)
|
||||
|
||||
|
||||
def test_the_scan_sees_a_sentence_when_there_is_one():
|
||||
"""A guard that matches nothing is green over nothing."""
|
||||
sample = ast.parse('HTTPException(status_code=400, detail=f"bad {x}")').body[0].value
|
||||
assert _is_text(sample.keywords[1].value)
|
||||
assert not _is_text(ast.parse('{"code": "refused", "reason": "x"}').body[0].value)
|
||||
@@ -79,7 +79,7 @@ def test_the_demo_entry_does_not_stamp_the_shared_account(client, db):
|
||||
|
||||
r = client.post("/auth/demo", json={"email": "visitor@example.com"})
|
||||
|
||||
assert r.json() == {"authenticated": True}
|
||||
assert r.json() == {"ok": True, "authenticated": True}
|
||||
demo = auth_routes.get_or_create_demo_user(db)
|
||||
db.refresh(demo)
|
||||
assert demo.last_login_at is None
|
||||
|
||||
@@ -140,9 +140,9 @@ def test_a_refused_token_refresh_names_the_oauth_reason_and_not_its_prose(sink,
|
||||
request=httpx.Request("POST", "https://oauth2.googleapis.com/token"),
|
||||
)
|
||||
client = yt.YouTubeClient.__new__(yt.YouTubeClient) # no HTTP client: `_send` is the stub
|
||||
client.db = None
|
||||
client.db = SimpleNamespace(add=lambda obj: None, commit=lambda: None)
|
||||
client.user = SimpleNamespace(
|
||||
token=SimpleNamespace(access_token=None, expiry=None, refresh_token_enc=b"x")
|
||||
id=1, token=SimpleNamespace(access_token=None, expiry=None, refresh_token_enc=b"x")
|
||||
)
|
||||
monkeypatch.setattr(yt, "decrypt_optional", lambda value: None)
|
||||
monkeypatch.setattr(yt, "decrypt", lambda value: "a-refresh-token")
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Dependency overrides go through `conftest.override_dep`, never `app.dependency_overrides` directly.
|
||||
|
||||
The hand-rolled shape — assign, then `app.dependency_overrides.clear()` in a `finally` — wipes EVERY
|
||||
override, not the ones the test set, so it stops fixtures composing: one test's teardown silently
|
||||
clobbers an override another fixture installed. `override_dep` restores each key's prior value
|
||||
instead. 22 files had drifted to the hand-rolled shape (it was the house norm at ~20 files to 3), and
|
||||
the sweep that converted them is only as good as the check that keeps it converted — this one.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
TESTS = Path(__file__).parent
|
||||
|
||||
|
||||
def test_no_test_writes_dependency_overrides_directly():
|
||||
offenders = [
|
||||
f"{p.name}:{n}"
|
||||
for p in sorted(TESTS.rglob("*.py"))
|
||||
if p.name not in {"conftest.py", Path(__file__).name}
|
||||
for n, line in enumerate(p.read_text(encoding="utf-8").splitlines(), 1)
|
||||
if "dependency_overrides" in line
|
||||
]
|
||||
assert offenders == [], "use the override_dep fixture: " + ", ".join(offenders)
|
||||
|
||||
|
||||
def test_a_request_gets_the_session_that_checks_its_commit():
|
||||
"""`get_db` is overridden with `db.info["request"]` — the test session plus the check that the
|
||||
handler committed what it flushed (conftest `_request_dep`). A bare `lambda: db` hands the route
|
||||
the same session without the check, and a handler that forgot its commit passes again."""
|
||||
offenders = [
|
||||
f"{p.name}:{n}"
|
||||
for p in sorted(TESTS.rglob("*.py"))
|
||||
if p.name != Path(__file__).name
|
||||
for n, line in enumerate(p.read_text(encoding="utf-8").splitlines(), 1)
|
||||
if "override_dep(get_db," in line and 'override_dep(get_db, db.info["request"])' not in line
|
||||
]
|
||||
assert offenders == [], 'override get_db with db.info["request"]: ' + ", ".join(offenders)
|
||||
|
||||
|
||||
def test_the_commit_check_counts_only_the_root_transaction(db):
|
||||
"""A savepoint released inside a handler commits nothing to the database; ending it must not
|
||||
clear the check (round 2). Driven directly, as a request would drive the dependency."""
|
||||
import pytest
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
from app.models import User
|
||||
|
||||
conn = HTTPConnection(
|
||||
{"type": "http", "method": "POST", "path": "/probe", "headers": [], "query_string": b""}
|
||||
)
|
||||
request = db.info["request"](conn)
|
||||
session = next(request)
|
||||
session.add(User(email="savepoint-probe@example.com"))
|
||||
session.flush()
|
||||
with session.begin_nested():
|
||||
pass
|
||||
with pytest.raises(StopIteration):
|
||||
next(request)
|
||||
assert db.info["unfinished"] == ["POST /probe"]
|
||||
db.info["unfinished"].clear() # recorded as it should be — not this test's failure
|
||||
|
||||
|
||||
def test_a_request_that_raises_leaves_nothing_it_did_not_commit(db):
|
||||
"""Production closes a failing request's session, which rolls back whatever it flushed and did
|
||||
not commit; the shared test session must do the same, or a later read in the test sees a write
|
||||
production would have lost (E16: the tracker's issue link, kept only by an explicit commit in
|
||||
the handler's failure path). Driven directly, as a request would drive the dependency."""
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
from app.models import User
|
||||
|
||||
conn = HTTPConnection(
|
||||
{"type": "http", "method": "POST", "path": "/probe", "headers": [], "query_string": b""}
|
||||
)
|
||||
request = db.info["request"](conn)
|
||||
session = next(request)
|
||||
session.add(User(email="rolled-back-probe@example.com"))
|
||||
session.flush()
|
||||
with pytest.raises(RuntimeError):
|
||||
request.throw(RuntimeError("the handler failed"))
|
||||
assert db.scalar(select(User).where(User.email == "rolled-back-probe@example.com")) is None
|
||||
assert db.info["unfinished"] == [] # a failing request is rolled back, not reported
|
||||
@@ -0,0 +1,105 @@
|
||||
"""E28 S3e2 — adding a video AT a position: the list page's "Undo" of a removal.
|
||||
|
||||
The row has to come back where it was in ONE request. The two-call shape (append, then send the
|
||||
whole order) leaves it at the end of the list whenever the second call fails, and the reorder would
|
||||
also overwrite any change another tab made in between. Positions can have gaps after removals, so
|
||||
the position is an index over the ORDERED rows, and the list is renumbered around it.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.models import Channel, Playlist, PlaylistItem, Video
|
||||
|
||||
|
||||
def _seed(db, user, ids, *, positions=None, **kw):
|
||||
db.add(Channel(id="UCtest", title="Test"))
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in [*ids, "vX"]:
|
||||
db.add(Video(id=v, channel_id="UCtest", published_at=now))
|
||||
pl = Playlist(user_id=user.id, name="L", **kw)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for i, vid in enumerate(ids):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=(positions or range(99))[i]))
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
def _ids(db, pl_id):
|
||||
db.expire_all()
|
||||
return [
|
||||
r.video_id
|
||||
for r in db.execute(
|
||||
PlaylistItem.__table__.select()
|
||||
.where(PlaylistItem.playlist_id == pl_id)
|
||||
.order_by(PlaylistItem.position)
|
||||
).all()
|
||||
]
|
||||
|
||||
|
||||
def _add(client, pl_id, body):
|
||||
return client.post(f"/api/playlists/{pl_id}/items", json=body)
|
||||
|
||||
|
||||
def test_a_position_puts_the_video_back_in_its_place(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"])
|
||||
r = _add(as_user(user), pl.id, {"video_id": "vX", "position": 1})
|
||||
assert r.status_code == 200
|
||||
assert _ids(db, pl.id) == ["a", "vX", "b", "c"]
|
||||
|
||||
|
||||
def test_the_position_counts_rows_not_stored_numbers(db, make_user, as_user):
|
||||
# A removal leaves a gap (0, 5, 9): index 2 is before the THIRD row, not "position 2".
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"], positions=[0, 5, 9])
|
||||
_add(as_user(user), pl.id, {"video_id": "vX", "position": 2})
|
||||
assert _ids(db, pl.id) == ["a", "b", "vX", "c"]
|
||||
|
||||
|
||||
def test_a_position_past_the_end_appends(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b"])
|
||||
_add(as_user(user), pl.id, {"video_id": "vX", "position": 40})
|
||||
assert _ids(db, pl.id) == ["a", "b", "vX"]
|
||||
|
||||
|
||||
def test_no_position_still_appends(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b"])
|
||||
_add(as_user(user), pl.id, {"video_id": "vX"})
|
||||
assert _ids(db, pl.id) == ["a", "b", "vX"]
|
||||
|
||||
|
||||
def test_a_video_already_in_the_list_does_not_move(db, make_user, as_user):
|
||||
# The add is idempotent; a position must not turn it into a reorder.
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"])
|
||||
_add(as_user(user), pl.id, {"video_id": "c", "position": 0})
|
||||
assert _ids(db, pl.id) == ["a", "b", "c"]
|
||||
|
||||
|
||||
def test_a_negative_position_is_refused(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a"])
|
||||
r = _add(as_user(user), pl.id, {"video_id": "vX", "position": -1})
|
||||
assert r.status_code == 422
|
||||
assert _ids(db, pl.id) == ["a"]
|
||||
|
||||
|
||||
def test_putting_a_removed_video_back_leaves_a_linked_list_clean(db, make_user, as_user):
|
||||
# Remove + undo is no change at all: a YouTube-linked list must not stay "unsynced" after it.
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"], yt_playlist_id="PLx")
|
||||
client = as_user(user)
|
||||
from app.sync.playlists import current_fingerprint
|
||||
|
||||
pl.synced_fingerprint = current_fingerprint(db, pl)
|
||||
pl.dirty = False
|
||||
db.commit()
|
||||
client.delete(f"/api/playlists/{pl.id}/items/b")
|
||||
db.expire_all()
|
||||
assert db.get(Playlist, pl.id).dirty is True
|
||||
_add(client, pl.id, {"video_id": "b", "position": 1})
|
||||
db.expire_all()
|
||||
assert db.get(Playlist, pl.id).dirty is False
|
||||
@@ -13,6 +13,9 @@ Two properties are load-bearing and neither is visible from the happy path alone
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from app.models import Channel, Playlist, PlaylistItem, Video
|
||||
|
||||
|
||||
@@ -235,6 +238,13 @@ def test_item_edits_bump_updated_at(db, make_user, as_user):
|
||||
assert db.get(Playlist, src.id).updated_at > old
|
||||
assert db.get(Playlist, dst.id).updated_at > old
|
||||
|
||||
# …and removing one, the fourth item-only write (it had no test: E10 review coverage debt).
|
||||
src.updated_at = old
|
||||
db.commit()
|
||||
assert client.delete(f"/api/playlists/{src.id}/items/v1").status_code == 200
|
||||
db.expire_all()
|
||||
assert db.get(Playlist, src.id).updated_at > old
|
||||
|
||||
|
||||
def test_the_list_endpoint_publishes_updated_at(db, make_user, as_user):
|
||||
user = make_user()
|
||||
@@ -242,3 +252,43 @@ def test_the_list_endpoint_publishes_updated_at(db, make_user, as_user):
|
||||
rows = as_user(user).get("/api/playlists").json()
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["updated_at"] # the rail cannot sort by a field the API never sends
|
||||
|
||||
|
||||
def test_a_target_that_already_had_it_is_not_recently_changed(db, make_user, as_user):
|
||||
"""Moving into a list that already holds the video only removes it from the source: the
|
||||
target did not change, and bumping it promoted an untouched list in the rail's sort."""
|
||||
user = make_user()
|
||||
_seed_videos(db, ["v1"])
|
||||
src = _playlist(db, user, "Source", ["v1"])
|
||||
dst = _playlist(db, user, "Target", ["v1"])
|
||||
old = datetime.now(timezone.utc) - timedelta(days=3)
|
||||
src.updated_at = old
|
||||
dst.updated_at = old
|
||||
db.commit()
|
||||
|
||||
assert _move(as_user(user), src.id, "v1", dst.id).status_code == 200
|
||||
db.expire_all()
|
||||
assert db.get(Playlist, src.id).updated_at > old
|
||||
assert db.get(Playlist, dst.id).updated_at == old
|
||||
|
||||
|
||||
def test_no_add_can_land_in_the_target_while_a_move_holds_it(db, make_user):
|
||||
"""The already-in-target check is a read; it holds because the move locks both lists first and
|
||||
an insert into the target — another tab's add, a YouTube merge — needs the target row for its
|
||||
foreign key, so it waits for the move's commit instead of hitting uq_playlist_video under it."""
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.routes import playlists as plroutes
|
||||
|
||||
user = make_user()
|
||||
_seed_videos(db, ["v1"])
|
||||
src = _playlist(db, user, "Source", ["v1"])
|
||||
dst = _playlist(db, user, "Target", [])
|
||||
plroutes.lock_playlists(db, src.id, dst.id) # what the move holds until its commit
|
||||
with SessionLocal() as other:
|
||||
other.execute(text("SET LOCAL lock_timeout = '300ms'"))
|
||||
other.add(PlaylistItem(playlist_id=dst.id, video_id="v1", position=0))
|
||||
with pytest.raises(OperationalError, match="lock timeout"):
|
||||
other.flush()
|
||||
db.rollback()
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
"""E28 review — every request that reads a playlist's item positions and writes new ones locks the
|
||||
playlist row first.
|
||||
|
||||
Without it two adds (two tabs; the rail and the page) both read the same `max(position)` and land
|
||||
on one position, and a reorder racing an add collides with it — the list's order between the two
|
||||
rows is then whatever the planner returns. A true two-connection race is not reproducible on
|
||||
demand, so the test holds the mechanism: the parent row is taken `FOR UPDATE` by each such route.
|
||||
A Plex reorder that omits a row (added in another tab meanwhile) must not collide with it either.
|
||||
|
||||
Round 2: the lock also comes FIRST — before the request reads or writes a single item row. A remove
|
||||
that deleted an item and then touched the playlist beside a reorder that went the other way was a
|
||||
deadlock (a 500 for one of them), and an add that checked "already there?" before the lock let two
|
||||
adds of one video both insert (a 500 on the unique key).
|
||||
"""
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import event
|
||||
|
||||
from app.models import (
|
||||
Channel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
PlexPlaylist,
|
||||
PlexPlaylistItem,
|
||||
Video,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def locks(db):
|
||||
"""The tables each request took a row lock on (`SELECT … FROM <t> … FOR UPDATE`)."""
|
||||
seen: list[str] = []
|
||||
|
||||
def _record(conn, cursor, statement, *_):
|
||||
if "FOR UPDATE" in statement:
|
||||
seen.extend(re.findall(r"FROM\s+(\w+)", statement)[:1])
|
||||
|
||||
engine = db.get_bind()
|
||||
event.listen(engine, "before_cursor_execute", _record)
|
||||
yield seen
|
||||
event.remove(engine, "before_cursor_execute", _record)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def statements(db):
|
||||
"""Every statement a request ran, in order."""
|
||||
seen: list[str] = []
|
||||
|
||||
def _record(conn, cursor, statement, *_):
|
||||
seen.append(" ".join(statement.split()))
|
||||
|
||||
engine = db.get_bind()
|
||||
event.listen(engine, "before_cursor_execute", _record)
|
||||
yield seen
|
||||
event.remove(engine, "before_cursor_execute", _record)
|
||||
|
||||
|
||||
def _locked_before_items(seen: list[str], parent: str, items: str) -> str | None:
|
||||
"""The first statement touching `items` must come after a `FOR UPDATE` of `parent`; returns
|
||||
that lock statement (None when it came too late or never)."""
|
||||
lock = None
|
||||
for st in seen:
|
||||
if lock is None and "FOR UPDATE" in st and re.search(rf"FROM {parent}\b", st):
|
||||
lock = st
|
||||
if re.search(rf"\b{items}\b", st):
|
||||
return lock
|
||||
return lock
|
||||
|
||||
|
||||
def _yt(db, user):
|
||||
db.add(Channel(id="UCtest", title="Test"))
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in ("v1", "v2", "v3"):
|
||||
db.add(Video(id=v, channel_id="UCtest", published_at=now))
|
||||
pl = Playlist(user_id=user.id, name="L")
|
||||
other = Playlist(user_id=user.id, name="M")
|
||||
db.add_all([pl, other])
|
||||
db.flush()
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id="v1", position=0))
|
||||
db.commit()
|
||||
return pl.id, other.id
|
||||
|
||||
|
||||
def _plex(db, user):
|
||||
lib = PlexLibrary(plex_key="p1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
items = [PlexItem(rating_key=rk, library_id=lib.id, kind="movie", title=rk) for rk in "abcd"]
|
||||
db.add_all(items)
|
||||
pl = PlexPlaylist(user_id=user.id, title="L")
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for i, it in enumerate(items[:3]):
|
||||
db.add(PlexPlaylistItem(playlist_id=pl.id, item_id=it.id, position=i))
|
||||
db.commit()
|
||||
return pl.id
|
||||
|
||||
|
||||
YT = Playlist.__table__.name
|
||||
PLEX = PlexPlaylist.__table__.name
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call",
|
||||
[
|
||||
lambda c, pid, other: c.post(f"/api/playlists/{pid}/items", json={"video_id": "v2"}),
|
||||
lambda c, pid, other: c.post(
|
||||
f"/api/playlists/{pid}/items", json={"video_id": "v2", "position": 0}
|
||||
),
|
||||
lambda c, pid, other: c.post("/api/playlists/watch-later", json={"video_id": "v2"}),
|
||||
lambda c, pid, other: c.post(
|
||||
f"/api/playlists/{pid}/items/v1/move", json={"target_playlist_id": other}
|
||||
),
|
||||
lambda c, pid, other: c.put(f"/api/playlists/{pid}/order", json={"video_ids": ["v1"]}),
|
||||
],
|
||||
ids=["add", "add-at", "watch-later", "move", "reorder"],
|
||||
)
|
||||
def test_a_youtube_playlist_write_locks_the_playlist(db, make_user, as_user, locks, call):
|
||||
user = make_user()
|
||||
pid, other = _yt(db, user)
|
||||
r = call(as_user(user), pid, other)
|
||||
assert r.status_code == 200, r.text
|
||||
assert YT in locks
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call",
|
||||
[
|
||||
lambda c, pid: c.post(f"/api/plex/playlists/{pid}/items", json={"item_rating_key": "d"}),
|
||||
lambda c, pid: c.post(
|
||||
f"/api/plex/playlists/{pid}/items/bulk", json={"item_rating_keys": ["d"]}
|
||||
),
|
||||
lambda c, pid: c.post(
|
||||
f"/api/plex/playlists/{pid}/items/bulk",
|
||||
json={"item_rating_keys": ["d"], "positions": [0]},
|
||||
),
|
||||
lambda c, pid: c.put(
|
||||
f"/api/plex/playlists/{pid}/order", json={"item_rating_keys": ["c", "b", "a"]}
|
||||
),
|
||||
],
|
||||
ids=["add", "bulk", "bulk-at", "reorder"],
|
||||
)
|
||||
def test_a_plex_playlist_write_locks_the_playlist(db, make_user, as_user, locks, call):
|
||||
user = make_user()
|
||||
pid = _plex(db, user)
|
||||
r = call(as_user(user), pid)
|
||||
assert r.status_code == 200, r.text
|
||||
assert PLEX in locks
|
||||
|
||||
|
||||
def test_a_plex_reorder_that_omits_a_row_does_not_collide_with_it(db, make_user, as_user):
|
||||
"""The row "a" was added in another tab after this page loaded its order ["c", "b"]: it keeps
|
||||
its place after the given rows instead of sharing position 0 with "c"."""
|
||||
user = make_user()
|
||||
pid = _plex(db, user)
|
||||
r = as_user(user).put(f"/api/plex/playlists/{pid}/order", json={"item_rating_keys": ["c", "b"]})
|
||||
assert r.status_code == 200
|
||||
db.expire_all()
|
||||
rows = db.query(PlexPlaylistItem).filter_by(playlist_id=pid).all()
|
||||
by_id = {it.id: it.rating_key for it in db.query(PlexItem).all()}
|
||||
got = sorted((r.position, by_id[r.item_id]) for r in rows)
|
||||
assert got == [(0, "c"), (1, "b"), (2, "a")]
|
||||
|
||||
|
||||
YT_ITEMS = PlaylistItem.__table__.name
|
||||
PLEX_ITEMS = PlexPlaylistItem.__table__.name
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call",
|
||||
[
|
||||
lambda c, pid, other: c.post(f"/api/playlists/{pid}/items", json={"video_id": "v1"}),
|
||||
lambda c, pid, other: c.post(
|
||||
f"/api/playlists/{pid}/items", json={"video_id": "v2", "position": 0}
|
||||
),
|
||||
lambda c, pid, other: c.delete(f"/api/playlists/{pid}/items/v1"),
|
||||
lambda c, pid, other: c.post(
|
||||
f"/api/playlists/{pid}/items/v1/move", json={"target_playlist_id": other}
|
||||
),
|
||||
lambda c, pid, other: c.put(f"/api/playlists/{pid}/order", json={"video_ids": ["v1"]}),
|
||||
# the ORM cascade deletes the items before the list row
|
||||
lambda c, pid, other: c.delete(f"/api/playlists/{pid}"),
|
||||
],
|
||||
ids=["add-present", "add-at", "remove", "move", "reorder", "delete"],
|
||||
)
|
||||
def test_a_youtube_item_write_locks_the_playlist_before_any_item(
|
||||
db, make_user, as_user, statements, call
|
||||
):
|
||||
user = make_user()
|
||||
pid, other = _yt(db, user)
|
||||
c = as_user(user)
|
||||
statements.clear()
|
||||
r = call(c, pid, other)
|
||||
assert r.status_code == 200, r.text
|
||||
assert _locked_before_items(statements, YT, YT_ITEMS), statements
|
||||
|
||||
|
||||
def test_a_move_locks_both_lists_in_one_ordered_statement(db, make_user, as_user, statements):
|
||||
"""Two moves A→B and B→A each locking "its" list first would lock crosswise."""
|
||||
user = make_user()
|
||||
pid, other = _yt(db, user)
|
||||
statements.clear()
|
||||
r = as_user(user).post(
|
||||
f"/api/playlists/{pid}/items/v1/move", json={"target_playlist_id": other}
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
lock = _locked_before_items(statements, YT, YT_ITEMS)
|
||||
assert lock and "ORDER BY" in lock, lock
|
||||
# BOTH lists in that one statement: one bound id per list in its IN (…) (a single-id lock
|
||||
# renders the same IN and ORDER BY)
|
||||
listed = re.search(r" IN \((.*?)\) ORDER BY", lock)
|
||||
assert listed and len(re.findall(r"%\(\w+\)s", listed.group(1))) == 2, lock
|
||||
|
||||
|
||||
def test_watch_later_add_and_remove_lock_it_before_any_item(db, make_user, as_user, statements):
|
||||
user = make_user()
|
||||
_yt(db, user)
|
||||
c = as_user(user)
|
||||
assert c.post("/api/playlists/watch-later", json={"video_id": "v2"}).status_code == 200
|
||||
statements.clear()
|
||||
assert c.post("/api/playlists/watch-later", json={"video_id": "v2"}).status_code == 200
|
||||
assert _locked_before_items(statements, YT, YT_ITEMS), statements
|
||||
statements.clear()
|
||||
assert c.delete("/api/playlists/watch-later/v2").status_code == 200
|
||||
assert _locked_before_items(statements, YT, YT_ITEMS), statements
|
||||
|
||||
|
||||
def test_a_rename_locks_the_playlist(db, make_user, as_user, locks):
|
||||
"""A rename moves `dirty`, which the scheduled mirror refresh re-reads under this lock."""
|
||||
user = make_user()
|
||||
pid, _ = _yt(db, user)
|
||||
r = as_user(user).patch(f"/api/playlists/{pid}", json={"name": "New"})
|
||||
assert r.status_code == 200, r.text
|
||||
assert YT in locks
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call",
|
||||
[
|
||||
lambda c, pid: c.delete(f"/api/plex/playlists/{pid}/items/a"),
|
||||
lambda c, pid: c.post(
|
||||
f"/api/plex/playlists/{pid}/items/remove-bulk", json={"item_rating_keys": ["a", "b"]}
|
||||
),
|
||||
lambda c, pid: c.post(f"/api/plex/playlists/{pid}/items", json={"item_rating_key": "a"}),
|
||||
],
|
||||
ids=["remove", "remove-bulk", "add-present"],
|
||||
)
|
||||
def test_a_plex_item_write_locks_the_playlist_before_any_item(
|
||||
db, make_user, as_user, statements, call
|
||||
):
|
||||
user = make_user()
|
||||
pid = _plex(db, user)
|
||||
c = as_user(user)
|
||||
statements.clear()
|
||||
r = call(c, pid)
|
||||
assert r.status_code == 200, r.text
|
||||
assert _locked_before_items(statements, PLEX, PLEX_ITEMS), statements
|
||||
@@ -0,0 +1,210 @@
|
||||
"""E28 review C1 — one YouTube sync of a playlist at a time, whoever starts it.
|
||||
|
||||
A push reads YouTube and then writes both sides for seconds, committing step by step, so the
|
||||
client's in-flight flag (one page's menus) and a row lock (gone at the first commit) both leave two
|
||||
tabs, or a push and the scheduled mirror refresh, free to interleave. The claim is an advisory lock
|
||||
on a connection of its own; the test holds it from outside, the way a second request would.
|
||||
"""
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.auth import require_write_scope
|
||||
from app.db import SessionLocal
|
||||
from app.models import Playlist
|
||||
from app.routes import playlists as plroutes
|
||||
from app.sync import playlists as plsync
|
||||
|
||||
|
||||
def _linked(db, user, *, source="local"):
|
||||
pl = Playlist(user_id=user.id, name="L", source=source, yt_playlist_id="PL123")
|
||||
db.add(pl)
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
class FakeYT:
|
||||
"""YouTube as far as these routes read it: two playlists, none with items."""
|
||||
|
||||
def __init__(self, *_):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
return False
|
||||
|
||||
def iter_my_playlists(self):
|
||||
return [{"id": "PL123", "title": "Renamed"}, {"id": "PL999", "title": "New"}]
|
||||
|
||||
def iter_my_playlist_video_ids(self, _ytid):
|
||||
return []
|
||||
|
||||
def iter_playlist_items_with_ids(self, _ytid):
|
||||
return []
|
||||
|
||||
def get_playlist_snippet(self, _ytid):
|
||||
return {}
|
||||
|
||||
|
||||
def test_push_and_revert_refuse_while_the_list_is_claimed(
|
||||
db, as_user, make_user, override_dep, monkeypatch
|
||||
):
|
||||
user = make_user()
|
||||
pl = _linked(db, user)
|
||||
override_dep(require_write_scope, lambda: user)
|
||||
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
||||
ran: list[str] = []
|
||||
monkeypatch.setattr(plroutes, "YouTubeClient", FakeYT)
|
||||
monkeypatch.setattr(plroutes, "plan_merge", lambda d, u, p, live: {"units_estimate": 0})
|
||||
monkeypatch.setattr(plroutes, "merge_playlist", lambda d, u, p, live: ran.append("push") or {})
|
||||
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: ran.append("revert") or {})
|
||||
c = as_user(user)
|
||||
|
||||
with plsync.sync_claim(db, pl.id) as free:
|
||||
assert free
|
||||
for path in ("push", "revert-youtube"):
|
||||
r = c.post(f"/api/playlists/{pl.id}/{path}")
|
||||
assert r.status_code == 409
|
||||
assert r.json()["detail"] == {"code": "playlist", "reason": "sync_busy"}
|
||||
# a delete too: a push mid-way would plan the deleted list as emptied (E28 round 4)
|
||||
r = c.delete(f"/api/playlists/{pl.id}")
|
||||
assert r.status_code == 409
|
||||
assert r.json()["detail"] == {"code": "playlist", "reason": "sync_busy"}
|
||||
assert ran == []
|
||||
assert db.get(Playlist, pl.id) is not None
|
||||
|
||||
# Released on the way out: the same list syncs again, and a claim is per list.
|
||||
assert c.post(f"/api/playlists/{pl.id}/push").status_code == 200
|
||||
assert c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code == 200
|
||||
assert ran == ["push", "revert"]
|
||||
|
||||
|
||||
def test_a_claim_is_released_when_its_block_raises(db, make_user):
|
||||
pl = _linked(db, make_user())
|
||||
try:
|
||||
with plsync.sync_claim(db, pl.id) as free:
|
||||
assert free
|
||||
raise RuntimeError("boom")
|
||||
except RuntimeError:
|
||||
pass
|
||||
# Asked of pg_locks, not by claiming again: a lock leaked on a pooled connection would be granted
|
||||
# again to that same connection (advisory locks are re-entrant per session).
|
||||
held = db.scalar(
|
||||
text("SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND objid = :id"),
|
||||
{"id": pl.id},
|
||||
)
|
||||
assert held == 0
|
||||
|
||||
|
||||
def test_a_claim_draws_no_connection_from_the_app_pool(db, make_user):
|
||||
"""A claim is held for a whole push beside the request's pooled session; drawn from the pool
|
||||
too, a burst of pushes ran it dry and stalled every other request for pool_timeout."""
|
||||
from app.db import engine
|
||||
|
||||
pl = _linked(db, make_user())
|
||||
before = engine.pool.checkedout()
|
||||
with plsync.sync_claim(db, pl.id) as free:
|
||||
assert free
|
||||
assert engine.pool.checkedout() == before
|
||||
|
||||
|
||||
def test_the_scheduled_refresh_skips_a_mirror_being_synced(db, make_user, monkeypatch):
|
||||
user = make_user()
|
||||
busy = _linked(db, user, source="youtube")
|
||||
replaced: list[int] = []
|
||||
|
||||
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plsync, "YouTubeClient", FakeYT)
|
||||
monkeypatch.setattr(
|
||||
plsync, "_replace_items_from_youtube", lambda d, p, ids: replaced.append(p.id)
|
||||
)
|
||||
|
||||
with plsync.sync_claim(db, busy.id):
|
||||
out = plsync.sync_user_playlists(db, user)
|
||||
|
||||
assert out == {"synced": 1}
|
||||
assert busy.id not in replaced and len(replaced) == 1
|
||||
db.refresh(busy)
|
||||
assert busy.name == "L"
|
||||
|
||||
|
||||
def test_refused_reverts_do_not_spend_the_pull_cooldown(
|
||||
db, as_user, make_user, override_dep, monkeypatch
|
||||
):
|
||||
"""Round 2: the cooldown (three pulls per ten minutes) is checked after the claim, so clicking
|
||||
Revert while a push runs is refused as busy without using up a pull."""
|
||||
user = make_user()
|
||||
pl = _linked(db, user)
|
||||
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: {})
|
||||
c = as_user(user)
|
||||
with plsync.sync_claim(db, pl.id):
|
||||
for _ in range(4):
|
||||
assert c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code == 409
|
||||
assert c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code == 200
|
||||
|
||||
|
||||
def test_a_bulk_pull_runs_once_per_user_at_a_time(db, as_user, make_user, monkeypatch):
|
||||
"""Two bulk pulls at once both saw a YouTube list unmirrored and each created its mirror."""
|
||||
user = make_user()
|
||||
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plsync, "YouTubeClient", FakeYT)
|
||||
monkeypatch.setattr(plsync, "_replace_items_from_youtube", lambda d, p, ids: None)
|
||||
c = as_user(user)
|
||||
with plsync.user_sync_claim(db, user.id):
|
||||
assert plsync.sync_user_playlists(db, user) == {"synced": 0, "reason": "busy"}
|
||||
for _ in range(4):
|
||||
r = c.post("/api/playlists/sync-youtube")
|
||||
assert r.status_code == 409
|
||||
assert r.json()["detail"] == {"code": "playlist", "reason": "sync_all_busy"}
|
||||
assert c.post("/api/playlists/sync-youtube").json() == {"ok": True, "synced": 2}
|
||||
|
||||
|
||||
def test_a_gone_mirror_is_deleted_even_when_no_list_refreshes(db, make_user, monkeypatch):
|
||||
"""Every remaining mirror has unpushed edits, so nothing below the deletion commits."""
|
||||
user = make_user()
|
||||
gone = Playlist(user_id=user.id, name="Gone", source="youtube", yt_playlist_id="PLgone")
|
||||
kept = Playlist(user_id=user.id, name="K", source="youtube", yt_playlist_id="PL123", dirty=True)
|
||||
other = Playlist(
|
||||
user_id=user.id, name="O", source="youtube", yt_playlist_id="PL999", dirty=True
|
||||
)
|
||||
db.add_all([gone, kept, other])
|
||||
db.commit()
|
||||
gone_id = gone.id
|
||||
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plsync, "YouTubeClient", FakeYT)
|
||||
|
||||
assert plsync.sync_user_playlists(db, user) == {"synced": 0}
|
||||
db.rollback() # what the request's session close does to an uncommitted deletion
|
||||
assert db.get(Playlist, gone_id) is None
|
||||
|
||||
|
||||
def test_a_mirror_edited_during_the_pull_is_not_replaced(db, make_user, monkeypatch):
|
||||
"""The loop's `dirty` is a snapshot from before the YouTube calls; a remove committed by another
|
||||
request meanwhile must win over YouTube's order."""
|
||||
user = make_user()
|
||||
pl = _linked(db, user, source="youtube")
|
||||
replaced: list[int] = []
|
||||
|
||||
class EditedMeanwhile(FakeYT):
|
||||
def iter_my_playlists(self):
|
||||
return [{"id": "PL123", "title": "Renamed"}]
|
||||
|
||||
def iter_my_playlist_video_ids(self, _ytid):
|
||||
with SessionLocal() as other:
|
||||
other.get(Playlist, pl.id).dirty = True
|
||||
other.commit()
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plsync, "YouTubeClient", EditedMeanwhile)
|
||||
monkeypatch.setattr(
|
||||
plsync, "_replace_items_from_youtube", lambda d, p, ids: replaced.append(p.id)
|
||||
)
|
||||
|
||||
assert plsync.sync_user_playlists(db, user) == {"synced": 0}
|
||||
assert replaced == []
|
||||
db.expire_all()
|
||||
assert db.get(Playlist, pl.id).name == "L"
|
||||
@@ -0,0 +1,292 @@
|
||||
"""E28 review round 3 — the YouTube sync paths take the playlist row BEFORE they touch an item, and
|
||||
hold it to the commit that writes them.
|
||||
|
||||
Round 2 made every route edit lock the playlist first. The sync paths were left item-first (a push's
|
||||
and a re-pull's item replace, a delete's cascade), so a push beside a remove could deadlock (a 500),
|
||||
and the scheduled mirror refresh re-ingested inside its lock — its commit let go of the lock before
|
||||
the replace, so a remove made then was undone and the list read as synced. A push also replaced the
|
||||
items from a snapshot read before its YouTube calls, and marked the list clean over an edit made
|
||||
during its YouTube writes. The "other tab" here is a second session that commits while the push is
|
||||
talking to YouTube — when the push holds no lock, as a real one does not.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import delete, event, text
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.models import Channel, Playlist, PlaylistItem, Video
|
||||
from app.sync import playlists as pl_mod
|
||||
|
||||
ITEMS = PlaylistItem.__table__.name
|
||||
LISTS = Playlist.__table__.name
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def trace(db):
|
||||
"""Every statement and every COMMIT on the test engine, in order."""
|
||||
seen: list[str] = []
|
||||
|
||||
def _stmt(conn, cursor, statement, *_):
|
||||
seen.append(" ".join(statement.split()))
|
||||
|
||||
def _commit(conn):
|
||||
seen.append("COMMIT")
|
||||
|
||||
engine = db.get_bind()
|
||||
event.listen(engine, "before_cursor_execute", _stmt)
|
||||
event.listen(engine, "commit", _commit)
|
||||
yield seen
|
||||
event.remove(engine, "before_cursor_execute", _stmt)
|
||||
event.remove(engine, "commit", _commit)
|
||||
|
||||
|
||||
def _held_to_the_item_write(seen: list[str], verb: str) -> bool:
|
||||
"""The first `<verb> … playlist_items` comes after a `FOR UPDATE` of playlists, with no COMMIT
|
||||
in between (a commit ends the lock)."""
|
||||
locked = False
|
||||
for st in seen:
|
||||
if "FOR UPDATE" in st and re.search(rf"FROM {LISTS}\b", st):
|
||||
locked = True
|
||||
elif st == "COMMIT":
|
||||
locked = False
|
||||
elif st.startswith(verb) and re.search(rf"\b{ITEMS}\b", st):
|
||||
return locked
|
||||
return False
|
||||
|
||||
|
||||
class FakeYT:
|
||||
"""YouTube for the sync paths: one playlist, its items, and ids the Data API will not return."""
|
||||
|
||||
def __init__(self, items, *, unavailable=(), title="Mix", on_get=None, on_add=None):
|
||||
self.items = [(f"it{i}", v) for i, v in enumerate(items)]
|
||||
self.unavailable = set(unavailable)
|
||||
self.title = title
|
||||
self.get_calls = 0
|
||||
self._on_get = on_get
|
||||
self._on_add = on_add
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
return False
|
||||
|
||||
def iter_my_playlists(self):
|
||||
return [{"id": "PLyt", "title": self.title}]
|
||||
|
||||
def iter_my_playlist_video_ids(self, _pid):
|
||||
return [v for _, v in self.items]
|
||||
|
||||
def iter_playlist_items_with_ids(self, _pid):
|
||||
return [{"item_id": i, "video_id": v} for i, v in self.items]
|
||||
|
||||
def get_playlist_snippet(self, _pid):
|
||||
return {"title": self.title}
|
||||
|
||||
def get_videos(self, ids):
|
||||
self.get_calls += 1
|
||||
if self._on_get:
|
||||
self._on_get()
|
||||
return [
|
||||
{"id": v, "snippet": {"channelId": "UCtest", "publishedAt": "2020-01-01T00:00:00Z"}}
|
||||
for v in ids
|
||||
if v not in self.unavailable
|
||||
]
|
||||
|
||||
def create_playlist(self, title, *_, **__):
|
||||
return "PLyt"
|
||||
|
||||
def update_playlist_title(self, _pid, title):
|
||||
self.title = title
|
||||
|
||||
def add_playlist_item(self, _pid, vid, position=None):
|
||||
if self._on_add:
|
||||
self._on_add()
|
||||
iid = f"it{len(self.items) + 100}"
|
||||
self.items.append((iid, vid))
|
||||
return iid
|
||||
|
||||
def delete_playlist_item(self, item_id):
|
||||
self.items = [r for r in self.items if r[0] != item_id]
|
||||
|
||||
def move_playlist_item(self, item_id, _pid, _vid, position):
|
||||
row = next(r for r in self.items if r[0] == item_id)
|
||||
self.items.remove(row)
|
||||
self.items.insert(position, row)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_yt(monkeypatch):
|
||||
def install(fake):
|
||||
monkeypatch.setattr(pl_mod, "YouTubeClient", lambda *_: fake)
|
||||
return fake
|
||||
|
||||
return install
|
||||
|
||||
|
||||
def _seed(db, ids):
|
||||
db.add(Channel(id="UCtest", title="Test"))
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in ids:
|
||||
db.add(Video(id=v, channel_id="UCtest", published_at=now))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _playlist(db, user, local, *, source="local", baseline=None):
|
||||
pl = Playlist(
|
||||
user_id=user.id,
|
||||
name="Mix",
|
||||
source=source,
|
||||
yt_playlist_id="PLyt",
|
||||
synced_video_ids=None if baseline is None else pl_mod._dump_ids(baseline),
|
||||
)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for pos, v in enumerate(local):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos))
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
def _other_tab(pl_id, *, add=None, remove=None, rename=None, drop=False):
|
||||
"""A second session's edit, committed at once — what another tab's request does. It runs on
|
||||
the push's own thread: a lock the push wrongly held here would be waited on forever, so the
|
||||
wait is bounded and a regression fails instead of hanging the gate."""
|
||||
with SessionLocal() as s:
|
||||
s.execute(text("SET LOCAL lock_timeout = '2s'"))
|
||||
if drop:
|
||||
s.execute(delete(Playlist).where(Playlist.id == pl_id))
|
||||
if rename:
|
||||
s.get(Playlist, pl_id).name = rename
|
||||
if add:
|
||||
pos = len(s.query(PlaylistItem).filter_by(playlist_id=pl_id).all())
|
||||
s.add(PlaylistItem(playlist_id=pl_id, video_id=add, position=pos))
|
||||
if remove:
|
||||
s.query(PlaylistItem).filter_by(playlist_id=pl_id, video_id=remove).delete()
|
||||
s.commit()
|
||||
|
||||
|
||||
def test_the_mirror_refresh_holds_its_lock_through_a_list_with_a_private_video(
|
||||
db, make_user, fake_yt, trace
|
||||
):
|
||||
user = make_user()
|
||||
_seed(db, ["a"])
|
||||
pl = _playlist(db, user, ["a"], source="youtube")
|
||||
fake = fake_yt(FakeYT(["a", "ghost"], unavailable={"ghost"}))
|
||||
trace.clear()
|
||||
pl_mod.pull_user_playlists(db, user)
|
||||
assert _held_to_the_item_write(trace, "DELETE"), trace
|
||||
# the private video is asked for once, not again under the lock
|
||||
assert fake.get_calls == 1
|
||||
assert pl_mod._local_video_ids(db, pl) == ["a"]
|
||||
|
||||
|
||||
def test_the_mirror_drop_locks_the_list_before_its_items(db, make_user, fake_yt, trace):
|
||||
user = make_user()
|
||||
_seed(db, ["a"])
|
||||
pl = _playlist(db, user, ["a"], source="youtube")
|
||||
pl.yt_playlist_id = "PLgone"
|
||||
db.commit()
|
||||
fake_yt(FakeYT([]))
|
||||
trace.clear()
|
||||
pl_mod.pull_user_playlists(db, user)
|
||||
assert _held_to_the_item_write(trace, "DELETE"), trace
|
||||
|
||||
|
||||
def test_a_repull_ingests_before_its_lock_and_holds_it_to_the_replace(
|
||||
db, make_user, fake_yt, trace
|
||||
):
|
||||
user = make_user()
|
||||
_seed(db, ["a"])
|
||||
pl = _playlist(db, user, ["a"])
|
||||
fake = fake_yt(FakeYT(["b", "a", "ghost"], unavailable={"ghost"}, title="Renamed"))
|
||||
trace.clear()
|
||||
pl_mod.repull_playlist(db, user, pl)
|
||||
assert _held_to_the_item_write(trace, "DELETE"), trace
|
||||
assert fake.get_calls == 1
|
||||
db.refresh(pl)
|
||||
assert (pl.name, pl.dirty) == ("Renamed", False)
|
||||
assert pl_mod._local_video_ids(db, pl) == ["b", "a"]
|
||||
|
||||
|
||||
def test_a_push_locks_the_list_before_it_replaces_the_items(db, make_user, fake_yt, trace):
|
||||
user = make_user()
|
||||
_seed(db, ["a", "b"])
|
||||
pl = _playlist(db, user, ["a", "b"], baseline=["a"])
|
||||
fake_yt(FakeYT(["a", "y"]))
|
||||
trace.clear()
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
assert _held_to_the_item_write(trace, "DELETE"), trace
|
||||
|
||||
|
||||
def test_an_add_made_while_a_push_reads_youtube_survives_it(db, make_user, fake_yt):
|
||||
user = make_user()
|
||||
_seed(db, ["a", "x"])
|
||||
pl = _playlist(db, user, ["a"], baseline=["a"])
|
||||
# 'y' is new on YouTube, so the push ingests it — and the other tab adds 'x' meanwhile.
|
||||
fake = fake_yt(FakeYT(["a", "y"], on_get=lambda: _other_tab(pl.id, add="x")))
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
local = pl_mod._local_video_ids(db, pl)
|
||||
assert set(local) == {"a", "x", "y"}
|
||||
assert "x" in [v for _, v in fake.items] # and it reached YouTube as a local add
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linked", [True, False], ids=["linked", "first-push"])
|
||||
def test_a_rename_made_during_a_push_s_youtube_writes_stays_dirty(db, make_user, fake_yt, linked):
|
||||
"""The closing baseline is YouTube's title, not the one the lock re-reads: over the new name it
|
||||
matched the rename, and the list read synced with YouTube still on the old title."""
|
||||
user = make_user()
|
||||
_seed(db, ["a", "b"])
|
||||
pl = _playlist(db, user, ["a", "b"], baseline=["a"] if linked else None)
|
||||
if not linked:
|
||||
pl.yt_playlist_id = None
|
||||
db.commit()
|
||||
fake = fake_yt(
|
||||
FakeYT(["a"] if linked else [], on_add=lambda: _other_tab(pl.id, rename="Mix 2"))
|
||||
)
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
db.refresh(pl)
|
||||
assert (pl.name, fake.title) == ("Mix 2", "Mix")
|
||||
assert pl.dirty is True # the rename has not reached YouTube; the list must say so
|
||||
|
||||
|
||||
def test_a_push_stops_at_a_list_deleted_while_it_read_youtube(db, make_user, fake_yt):
|
||||
"""A delete landing mid-push left a plan over no items: every item "removed here", so the push
|
||||
emptied the YouTube playlist the delete meant to keep."""
|
||||
user = make_user()
|
||||
_seed(db, ["a", "b"])
|
||||
pl = _playlist(db, user, ["a", "b"])
|
||||
# a video YouTube would not return last time, retried now — the read the delete lands in
|
||||
pl.synced_video_ids = json.dumps({"ids": ["a", "b"], "unavailable": ["ghost"]})
|
||||
db.commit()
|
||||
fake = fake_yt(
|
||||
FakeYT(
|
||||
["a", "b", "ghost"], unavailable={"ghost"}, on_get=lambda: _other_tab(pl.id, drop=True)
|
||||
)
|
||||
)
|
||||
raised = None
|
||||
try:
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
except Exception as exc: # noqa: BLE001 — the regression raises too, AFTER its YouTube writes
|
||||
raised = exc
|
||||
assert [v for _, v in fake.items] == ["a", "b", "ghost"] # YouTube untouched
|
||||
assert isinstance(raised, pl_mod.PlaylistGone) # which the route answers with a 404
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linked", [True, False], ids=["linked", "first-push"])
|
||||
def test_an_edit_made_during_a_push_s_youtube_writes_stays_dirty(db, make_user, fake_yt, linked):
|
||||
user = make_user()
|
||||
_seed(db, ["a", "b"])
|
||||
pl = _playlist(db, user, ["a", "b"], baseline=["a"] if linked else None)
|
||||
if not linked:
|
||||
pl.yt_playlist_id = None
|
||||
db.commit()
|
||||
fake_yt(FakeYT(["a"] if linked else [], on_add=lambda: _other_tab(pl.id, remove="a")))
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
db.refresh(pl)
|
||||
assert pl_mod._local_video_ids(db, pl) == ["b"]
|
||||
assert pl.dirty is True # the removal has not reached YouTube; the list must say so
|
||||
@@ -42,7 +42,7 @@ def _seed(db, *, status: str, position: int):
|
||||
return it, user
|
||||
|
||||
|
||||
def _clear(db, user, it, monkeypatch) -> list[tuple]:
|
||||
def _clear(db, user, it, monkeypatch, override_dep) -> list[tuple]:
|
||||
"""Drive the endpoint with the Plex link satisfied and the background push recorded."""
|
||||
pushes: list[tuple] = []
|
||||
monkeypatch.setattr(state, "_configured_cache", True) # pass the instance-setup request gate
|
||||
@@ -52,32 +52,29 @@ def _clear(db, user, it, monkeypatch) -> list[tuple]:
|
||||
"push_state_to_plex",
|
||||
lambda user_id, item_id, rating_key, action, *rest: pushes.append((rating_key, action)),
|
||||
)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
resp = TestClient(app).post(f"/api/plex/item/{it.rating_key}/state", json={"status": "new"})
|
||||
assert resp.status_code == 200, resp.text
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: user)
|
||||
resp = TestClient(app).post(f"/api/plex/item/{it.rating_key}/state", json={"status": "new"})
|
||||
assert resp.status_code == 200, resp.text
|
||||
return pushes
|
||||
|
||||
|
||||
def test_clearing_an_in_progress_item_tells_plex(db, monkeypatch):
|
||||
def test_clearing_an_in_progress_item_tells_plex(db, monkeypatch, override_dep):
|
||||
# The regression: status "new" + a position IS the in-progress shape, and it must push.
|
||||
it, user = _seed(db, status="new", position=92)
|
||||
assert _clear(db, user, it, monkeypatch) == [("98614", "unwatched")]
|
||||
assert _clear(db, user, it, monkeypatch, override_dep) == [("98614", "unwatched")]
|
||||
assert db.query(PlexState).count() == 0
|
||||
|
||||
|
||||
def test_clearing_a_watched_item_still_tells_plex(db, monkeypatch):
|
||||
def test_clearing_a_watched_item_still_tells_plex(db, monkeypatch, override_dep):
|
||||
# The case the old guard did cover — kept, so the fix cannot narrow into a new hole.
|
||||
it, user = _seed(db, status="watched", position=0)
|
||||
assert _clear(db, user, it, monkeypatch) == [("98614", "unwatched")]
|
||||
assert _clear(db, user, it, monkeypatch, override_dep) == [("98614", "unwatched")]
|
||||
|
||||
|
||||
def test_unhiding_touches_plex_not_at_all(db, monkeypatch):
|
||||
def test_unhiding_touches_plex_not_at_all(db, monkeypatch, override_dep):
|
||||
# `hidden` is a Siftlode-only concept "with no Plex analogue, so it never touches Plex" — the
|
||||
# endpoint's own docstring. The old status-only guard pushed here anyway, un-marking an item on
|
||||
# Plex that the user had merely hidden.
|
||||
it, user = _seed(db, status="hidden", position=0)
|
||||
assert _clear(db, user, it, monkeypatch) == []
|
||||
assert _clear(db, user, it, monkeypatch, override_dep) == []
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user