Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
29c0715237 | ||
|
|
6b86f99249 | ||
|
|
3563858fd1 | ||
|
|
08e62bbb16 | ||
|
|
9454717f47 | ||
|
|
6ac9ffd45e | ||
|
|
fa55e01ef0 | ||
|
|
ab963c81d2 | ||
|
|
59e6cc027d | ||
|
|
ca94a716d8 | ||
|
|
63d01babe5 | ||
|
|
705377c797 | ||
|
|
2819049133 | ||
|
|
95b94a1d72 | ||
|
|
12077fbebc | ||
|
|
519fbe2a18 | ||
|
|
a39504e617 | ||
|
|
aac79ffa53 | ||
|
|
67c9f10153 | ||
|
|
584d082b6b | ||
|
|
f3a85916ec | ||
|
|
f6298d29af | ||
|
|
632e39d458 | ||
|
|
13d95ef70e | ||
|
|
0f9ed7cdbf | ||
|
|
ffcea0cb07 | ||
|
|
a3f8314c9a | ||
|
|
d5ccb1091a | ||
|
|
528774d640 | ||
|
|
3a9ea9e330 | ||
|
|
25f2d9d0e4 | ||
|
|
5ab6778b97 | ||
|
|
26697472cf | ||
|
|
2abd4ffcb6 | ||
|
|
a9ac303d3e | ||
|
|
740602b9bf | ||
|
|
e31a588f19 | ||
|
|
cda626be10 | ||
|
|
b7ce1d467c | ||
|
|
52fb999af6 | ||
|
|
e0453fa07d | ||
|
|
2525a12264 | ||
|
|
8e09dd85f8 | ||
|
|
addb3d1b32 | ||
|
|
86884e368e | ||
|
|
51f881681b | ||
|
|
27483ae1d6 | ||
|
|
595c29e17c | ||
|
|
acf93a1f36 | ||
|
|
84278293df | ||
|
|
dc24d80ed4 | ||
|
|
3810af66da | ||
|
|
eb339195c7 | ||
|
|
5d1065c1bc | ||
|
|
6ffb5bd8c1 | ||
|
|
dddf79ec2f | ||
|
|
5ebb4cb506 | ||
|
|
0b1d905866 | ||
|
|
52c15adaf2 | ||
|
|
3372626e34 | ||
|
|
98bb841a6a | ||
|
|
98bf0a9d66 | ||
|
|
17bb14fb18 | ||
|
|
70f98b352f | ||
|
|
efa4aeeeda | ||
|
|
a7d41eb5f3 | ||
|
|
76eead976f | ||
|
|
922d589b09 | ||
|
|
a975ab627a | ||
|
|
9bedd96774 | ||
|
|
383c85231a | ||
|
|
de62035889 | ||
|
|
b456232d00 | ||
|
|
c9e95ecea6 | ||
|
|
7987bd8a37 | ||
|
|
3411265dc1 | ||
|
|
098b8dbe43 | ||
|
|
f48f16830e | ||
|
|
955c23d321 | ||
|
|
d3e1d8219c | ||
|
|
4480839db0 | ||
|
|
8e25567ff2 | ||
|
|
8f65bd184e | ||
|
|
ae9c248987 | ||
|
|
bd6aaa1e43 | ||
|
|
2c65edfffd | ||
|
|
6c2c01e67b | ||
|
|
ef343dbec0 | ||
|
|
a58a398427 | ||
|
|
d510c8a534 | ||
|
|
dc5206f9d3 | ||
|
|
89e65ca331 | ||
|
|
17916b411a | ||
|
|
3ef2c38eb4 | ||
|
|
4c63d6660d | ||
|
|
a453b77f04 | ||
|
|
8e1004fe6f | ||
|
|
663f30feeb | ||
|
|
6653d81a80 | ||
|
|
21df7c8348 | ||
|
|
97381e65d0 | ||
|
|
f8ced6d819 | ||
|
|
f254588409 | ||
|
|
3e7c79fa69 | ||
|
|
47c12f9405 | ||
|
|
bd0914b7ed | ||
|
|
079cee4e9c | ||
|
|
edf2f22ef9 | ||
|
|
e1ef56cdbc | ||
|
|
4d0abc61d5 | ||
|
|
7c2b22bafa | ||
|
|
5e5c8a42a9 | ||
|
|
2b3094ffc5 | ||
|
|
9c91eb5d22 | ||
|
|
2df60fbf07 | ||
|
|
a2d17d693c | ||
|
|
c4917619ee | ||
|
|
760bdae278 | ||
|
|
1b946113c6 | ||
|
|
b0e51c6f89 | ||
|
|
7455fff401 | ||
|
|
aaf1843b09 | ||
|
|
a4737b92e4 | ||
|
|
75ae810866 | ||
|
|
9d7526ee57 | ||
|
|
fa0b0edd63 | ||
|
|
bfc6d199c0 | ||
|
|
4d6e59b80a | ||
|
|
4646aae320 | ||
|
|
eddeca82d3 | ||
|
|
69edeecc45 | ||
|
|
6088e476a7 | ||
|
|
ce171fa954 | ||
|
|
113da76c36 | ||
|
|
4ef11901db | ||
|
|
29abf609f3 | ||
|
|
e13d93a3ed | ||
|
|
b9850b1710 | ||
|
|
905a9f2856 | ||
|
|
1d8b83c37c | ||
|
|
62dbf35ea3 | ||
|
|
e9ccd9bc98 | ||
|
|
cc8e0648db | ||
|
|
ff80cc96fd | ||
|
|
295c1db5f1 | ||
|
|
0521149cf1 | ||
|
|
b4a6569d0d | ||
|
|
1833f594cb | ||
|
|
1615838f33 | ||
|
|
7fa8f4ebde | ||
|
|
6354454cc9 | ||
|
|
c47c193ad1 | ||
|
|
db10bfbbca | ||
|
|
dac9d89f91 | ||
|
|
81936e9f56 | ||
|
|
86f467bd65 | ||
|
|
7d480e2ad0 | ||
|
|
ad136c3fe8 | ||
|
|
9c4a999e54 | ||
|
|
3f5634a44f | ||
|
|
f4bc60d659 | ||
|
|
7055548260 | ||
|
|
477271907e | ||
|
|
50923bb684 | ||
|
|
d2e40dcffb | ||
|
|
58878958d8 | ||
|
|
b47d22cf3b | ||
|
|
e804ff3b8e | ||
|
|
cd6f119883 | ||
|
|
321776480e | ||
|
|
7c0647f87a | ||
|
|
efd7b217ca | ||
|
|
3d7ef9e945 | ||
|
|
7fce7a7ce5 | ||
|
|
f5018bdcbc | ||
|
|
b82c22b888 | ||
|
|
f839a461e6 | ||
|
|
8525666913 | ||
|
|
3f24248e01 | ||
|
|
6b86673af4 | ||
|
|
d924c665f8 | ||
|
|
aa51644f5f | ||
|
|
378b7559e4 | ||
|
|
05121fbc99 | ||
|
|
bf518223e5 | ||
|
|
332d24a646 | ||
|
|
c5e022f056 | ||
|
|
4065ac7e6a | ||
|
|
ce08fc2aec | ||
|
|
ea6d829c29 | ||
|
|
6a7ff330f6 | ||
|
|
7e1aeeb9fe | ||
|
|
7d682c90d5 | ||
|
|
cc9fc3b544 | ||
|
|
c9101e3450 | ||
|
|
51daf3e388 | ||
|
|
09d351a0e7 | ||
|
|
b5078003ce | ||
|
|
f2c32cfb8e | ||
|
|
1453fa4a33 | ||
|
|
efe106dfda | ||
|
|
ec63056bd7 | ||
|
|
09becb3d00 | ||
|
|
8d2ec5e867 | ||
|
|
0e9839e73d | ||
|
|
05ab963951 |
@@ -1,22 +1,50 @@
|
||||
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
||||
# independent of any one machine. Two parallel jobs match the two local lanes:
|
||||
# independent of any one machine. Two jobs match the two local lanes:
|
||||
# frontend — tsc (3 projects) + eslint + prettier --check + knip + vitest
|
||||
# backend — ruff + the full pytest suite (pure-logic + the DB-backed lane) against a real Postgres
|
||||
#
|
||||
# They are declared side by side but they DO NOT run in parallel: the lab has a single act_runner
|
||||
# with one slot, so they queue. Measured across runs #86-#91 — the backend job starts at the exact
|
||||
# second the frontend one ends, every time, and the pair costs ~4 + ~6 = **~10 minutes from the
|
||||
# push**. Anything that waits on this gate has to budget for the sum, not the max; the ops
|
||||
# `release` read-back timed out on its first real run for believing otherwise.
|
||||
#
|
||||
# Runner: the lab act_runner (LXC 115) mounts the Docker socket, so service containers work. The backend
|
||||
# runs pytest NATIVELY against a `postgres` service (not the local docker-compose.test.yml, whose
|
||||
# bind-mount is unreliable under a mounted-socket runner); fidelity is kept by installing the SAME pinned
|
||||
# requirements, Postgres 16, and ffmpeg, and by setting SIFTLODE_TEST_REQUIRE_DB=1 so a DB/ffmpeg that
|
||||
# failed to come up is a hard error, never a silent skip. node 22 matches the prod build image
|
||||
# (Dockerfile `node:22-alpine`).
|
||||
# failed to come up is a hard error, never a silent skip.
|
||||
#
|
||||
# The frontend container's node major is NOT free: it must equal the prod build image's and the dev
|
||||
# machine's, because vitest/tsc read the engine's own Intl and web-API implementations and those split
|
||||
# between majors. `.nvmrc` is the single source of truth and
|
||||
# `frontend/src/lib/nodeEngine.contract.test.ts` fails this gate if any of them drift apart.
|
||||
name: gate
|
||||
|
||||
on: [push, pull_request]
|
||||
# Branch pushes only. A release pushes `main` and then the `v<ver>` tag, and a bare `on: [push]`
|
||||
# treated the tag as a second push of the IDENTICAL commit — every release ran this whole gate
|
||||
# twice, to the same verdict, for nothing (runs #81-#90 are five such pairs). `branches: ['**']`
|
||||
# keeps every branch covered and drops tag refs.
|
||||
#
|
||||
# It has to be `branches`, NOT the `tags-ignore` this obviously wants to be. Gitea's matchPushEvent
|
||||
# requires EVERY named filter to match, and each one bails on the wrong ref kind: `tags-ignore`
|
||||
# starts with `if !refName.IsTag() { break }`, so a filter meant to skip tags rejects every BRANCH
|
||||
# push and the gate stops running entirely. `branches` bails the same way on a tag, which is the
|
||||
# direction we want. (Measured in go-gitea v1.27.3 modules/actions/workflows.go, not assumed — it
|
||||
# differs from GitHub, where `tags-ignore` alone leaves branches unfiltered.)
|
||||
#
|
||||
# If that filter were ever wrong the gate would not run at all — which is why `siftlode release`
|
||||
# now ends by reading the verdict back for the sha it pushed, and says so out loud when there is
|
||||
# no run for it. A gate that silently stops running is the failure this repo already lived through.
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
container: node:22-bookworm
|
||||
container: node:24-bookworm
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
@@ -34,7 +62,11 @@ jobs:
|
||||
# A combined python+node image: python 3.12 (matches the prod Dockerfile) for the tests, and node
|
||||
# so the JS actions (checkout) run. actions/setup-python is NOT usable here — the runner can't reach
|
||||
# GitHub's python-versions download, so it must be baked into the image.
|
||||
container: nikolaik/python-nodejs:python3.12-nodejs20
|
||||
#
|
||||
# This node runs the checkout action, never our code, so it is deliberately OUTSIDE the engine
|
||||
# contract the frontend job is held to — it tracks `.nvmrc` as hygiene (node 20 went EOL in April
|
||||
# 2026), but a missing upstream tag here must never be able to block the gate.
|
||||
container: nikolaik/python-nodejs:python3.12-nodejs24
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
@@ -61,6 +93,22 @@ jobs:
|
||||
run: python -m ruff check --config backend/ruff.toml backend scripts
|
||||
- name: ruff format --check
|
||||
run: python -m ruff format --check --config backend/ruff.toml backend scripts
|
||||
# The ninth lane (E27 S3). Run from backend/ so it resolves exactly as the local lane does —
|
||||
# pyrightconfig.json's include/extraPaths are relative to the config, but the baseline is
|
||||
# found relative to the working directory. New errors fail; the ~200 pre-existing ones are in
|
||||
# .basedpyright/baseline.json and burn down epic by epic. Matching is per (file, rule, column
|
||||
# span) plus a count, so a new error of the SAME shape in a file where a sibling was just
|
||||
# fixed is absorbed — burn the hot files down first.
|
||||
#
|
||||
# The second step is not redundant. basedpyright TIGHTENS the baseline in place when the count
|
||||
# drops, with no --writebaseline, and CI throws its workspace away — so without this a
|
||||
# burn-down is green here and red on the developer's `siftlode check`, which has the same
|
||||
# guard. Two gates over one tree must not answer differently.
|
||||
- name: basedpyright
|
||||
run: python -m basedpyright --project pyrightconfig.json
|
||||
working-directory: backend
|
||||
- name: basedpyright baseline is committed
|
||||
run: git diff --exit-code -- backend/.basedpyright/baseline.json
|
||||
- name: pytest
|
||||
run: python -m pytest
|
||||
working-directory: backend
|
||||
|
||||
@@ -37,3 +37,7 @@ backups/
|
||||
Thumbs.db
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# TypeScript incremental build artifacts — machine-local, absolute-path-bearing, and no gate lane
|
||||
# reads or regenerates them, so a committed one is stale from the commit that adds it.
|
||||
*.tsbuildinfo
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
3.12
|
||||
+838
-1361
File diff suppressed because it is too large
Load Diff
@@ -7,17 +7,18 @@ worse than no rule — if you change how the code works, change this file in the
|
||||
|
||||
Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a download center). It is a
|
||||
**modular monolith**: one Postgres, one FastAPI process (uvicorn + APScheduler), one worker process
|
||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 18 + react-query frontend with a router-free navigation
|
||||
provider.
|
||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 19 (with the React Compiler) + react-query frontend, with a
|
||||
router-free navigation provider.
|
||||
|
||||
## The gate — never merge or ship red
|
||||
|
||||
The local gate is **ruff (`check` + `format --check`) + typecheck (3 tsc projects) + eslint + prettier
|
||||
The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + typecheck (3 tsc projects) + eslint + prettier
|
||||
--check + knip + vitest + pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
||||
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
||||
|
||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` + `ruff format --check backend` and `pytest` (the DB-lane needs Postgres).
|
||||
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` and burn down epic by epic. Fixed some? Regenerate with `--writebaseline`.
|
||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||
|
||||
## Workflow
|
||||
@@ -45,11 +46,13 @@ how it gets built. Concretely:
|
||||
behind each option and the weight of the decision. These decision points are where the product owner
|
||||
engages with the build and learns from it, so make them understandable — a reasoned menu, not a bare one.
|
||||
- **Mockups for design/UX decisions.** For any visual or interaction choice, show mockup options to
|
||||
choose from before building.
|
||||
choose from before building. How the UI looks and behaves is decided in
|
||||
[`docs/DESIGN.md`](docs/DESIGN.md) (its variants render in the dev-only `/design-lab`): build to it,
|
||||
and change it only through its decision log.
|
||||
- **Challenge unsound requests.** If a request is technically ill-advised and you can back it with real
|
||||
arguments, push back or propose an alternative — don't comply blindly.
|
||||
- **Verify UI in the browser before calling it done.** tsc/unit/DOM tests miss visual and interaction
|
||||
bugs; do a real visual check of any UI change.
|
||||
bugs; do a real visual check of any UI change, and run DESIGN.md's four-step Conformance check.
|
||||
- **Be terse everywhere else.** Outside those decision points, lead with the answer or decision. During
|
||||
implementation and debugging especially: no restating the same point reworded, no filler.
|
||||
|
||||
@@ -57,7 +60,7 @@ how it gets built. Concretely:
|
||||
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||
@@ -74,29 +77,37 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
|
||||
## Canonical patterns — when you have a choice, pick THIS
|
||||
|
||||
| Area | Use | Not |
|
||||
| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------ |
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE action button | `ui/Button` — pick a `SIZES` rung | a hand-rolled `<button className="px-3 py-2 …">` |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
| Area | Use | Not |
|
||||
| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Request SIZE | nothing — `bodylimit` caps every route (SEC-20); need more? add a ceiling to `ROUTE_MAX_BODY_BYTES` | `capped_body` on a route that does not read raw bytes itself |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE button | `ui/Button` (a `SIZES` rung) · `IconButton` · `Pressable` (a clickable card/row) · `Chip` · `DragHandle` | a raw `<button>` or `FocusSafeButton` outside the control primitives (lint-enforced) |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE text field / checkbox / slider | `ui/Input` · `Textarea` · `InnerInput` in a `fieldClass` frame · `Checkbox`/`CheckRow` · `Slider` | a raw `<input>`/`<textarea>` anywhere but those primitives (lint-enforced) |
|
||||
| FE glass surface | a role: `SURFACE.panel` / `card` / `menu` / `popover` / `tooltip` (`ui/surfaces.ts`) | a bare `glass`/`glass-card`/`glass-menu` beside a hand-picked radius (lint-enforced) |
|
||||
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
||||
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
||||
| FE effect that must not re-subscribe | `useEffectEvent` (React 19.2+), else a latest-value ref | an excused dep list asserting the same thing in prose |
|
||||
| FE hook suppression | an inline `eslint-disable-next-line` NAMING the invariant | `eslint-suppressions.json` (deliberately absent — E27 S2) |
|
||||
|
||||
## House rules that bite (learned from real bugs)
|
||||
|
||||
|
||||
+3
-1
@@ -10,7 +10,9 @@ ARG GIT_SHA=unknown
|
||||
ARG BUILD_DATE=
|
||||
|
||||
# Stage 1: build the frontend SPA
|
||||
FROM node:22-alpine AS frontend
|
||||
# The major is pinned to `.nvmrc` and held there by frontend/src/lib/nodeEngine.contract.test.ts:
|
||||
# this image, the CI gate's image and the dev machine must be ONE engine (see that test for why).
|
||||
FROM node:24-alpine AS frontend
|
||||
WORKDIR /fe
|
||||
# Deps first, keyed on the lockfile, so editing app source never re-resolves node_modules.
|
||||
COPY frontend/package.json frontend/package-lock.json ./
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,27 @@
|
||||
"""add tagline_hu to plex_items + plex_shows — the HU overlay's tagline (E30 S1)
|
||||
|
||||
Revision ID: 0091_plex_tagline_hu
|
||||
Revises: 0090_debug_session_draft
|
||||
Create Date: 2026-09-29
|
||||
|
||||
The Hungarian overlay section lists a tagline beside the title and summary; the info page showed the
|
||||
English one to a Hungarian reader. NULL until the next overlay sync fills it (serving falls back to EN).
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0091_plex_tagline_hu"
|
||||
down_revision: str | None = "0090_debug_session_draft"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("plex_items", sa.Column("tagline_hu", sa.Text(), nullable=True))
|
||||
op.add_column("plex_shows", sa.Column("tagline_hu", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("plex_shows", "tagline_hu")
|
||||
op.drop_column("plex_items", "tagline_hu")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add biography_hu to plex_people — TMDB's Hungarian biography (E30 S4)
|
||||
|
||||
Revision ID: 0092_plex_person_biography_hu
|
||||
Revises: 0091_plex_tagline_hu
|
||||
Create Date: 2026-09-29
|
||||
|
||||
The person page showed a Hungarian reader the English biography although TMDB has a Hungarian one for
|
||||
most people. The person fetch now appends `translations` and keeps the Hungarian biography beside the
|
||||
English one. Already-enriched people are un-stamped, so their next page open refetches them once (the
|
||||
same lazy path as a first open) and fills it; until then the English biography stands in.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0092_plex_person_biography_hu"
|
||||
down_revision: str | None = "0091_plex_tagline_hu"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("plex_people", sa.Column("biography_hu", sa.Text(), nullable=True))
|
||||
op.execute(
|
||||
"UPDATE plex_people SET tmdb_enriched_at = NULL "
|
||||
"WHERE tmdb_person_id IS NOT NULL AND tmdb_enriched_at IS NOT NULL"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("plex_people", "biography_hu")
|
||||
@@ -0,0 +1,29 @@
|
||||
"""re-fetch linked people — "Known for" ranked by votes, without guest visits (E30 S6)
|
||||
|
||||
Revision ID: 0093_plex_person_known_for_votes
|
||||
Revises: 0092_plex_person_biography_hu
|
||||
Create Date: 2026-09-30
|
||||
|
||||
A person's "Known for" pool was their credits ranked by TMDB popularity, so talk-show visits as
|
||||
themselves crowded out the films (Tom Cruise: 14 of 15 were talk shows, none in the library). The pool is
|
||||
now ranked by vote count, without guest visits, with directed works, and deeper. Already-enriched people
|
||||
are un-stamped so the people_enrich job fetches them again; their biography stays on the row meanwhile.
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0093_plex_person_known_for_votes"
|
||||
down_revision: str | None = "0092_plex_person_biography_hu"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"UPDATE plex_people SET tmdb_enriched_at = NULL "
|
||||
"WHERE tmdb_person_id IS NOT NULL AND tmdb_enriched_at IS NOT NULL"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass # the watermark only says "fetched"; a refetch under the old ranking is not worth undoing to
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Hungarian genres on every Plex row, not only on overlaid ones (E30 review)
|
||||
|
||||
Revision ID: 0094_plex_genres_hu_everywhere
|
||||
Revises: 0093_plex_person_known_for_votes
|
||||
Create Date: 2026-09-30
|
||||
|
||||
The HU genre facets and filter read `genres_hu`, which only the overlay pass wrote — so a library with
|
||||
no HU twin, or a guid-less row, offered English genres a Hungarian filter could never match. The sync
|
||||
now writes `genres_hu` for every row (plex/sync.py `_apply_facet_fields`); this fills the rows it has
|
||||
not reached yet, from their own English list through the same map.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from app.plex.genres import hu_genres
|
||||
|
||||
revision: str = "0094_plex_genres_hu_everywhere"
|
||||
down_revision: str | None = "0093_plex_person_known_for_votes"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
for table in ("plex_items", "plex_shows"):
|
||||
rows = conn.execute(
|
||||
sa.text(
|
||||
f"SELECT id, genres FROM {table} "
|
||||
"WHERE genres_hu IS NULL AND jsonb_typeof(genres) = 'array'"
|
||||
)
|
||||
).fetchall()
|
||||
updates = [
|
||||
{"id": rid, "g": json.dumps(hu)} for rid, genres in rows if (hu := hu_genres(genres))
|
||||
]
|
||||
if updates:
|
||||
conn.execute(
|
||||
sa.text(f"UPDATE {table} SET genres_hu = CAST(:g AS jsonb) WHERE id = :id"), updates
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass # derived from `genres`; the sync writes the same values again
|
||||
@@ -23,7 +23,8 @@ Three layers, by *who* runs the code — not by feature:
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||
level) and `debugsession` (a recording that tags its user's lines, E16), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
||||
reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
module must never import a `routes/…` module).
|
||||
|
||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""The global request-body cap (SEC-20).
|
||||
|
||||
**The hole.** FastAPI reads a declared body IN FULL before auth or any other dependency runs — the
|
||||
body parameter has to be parsed to be passed in. So `POST /auth/password-login` drains a 32 MiB
|
||||
upload from an anonymous caller and only then answers 401. 70 of the 72 body-declaring routes behave
|
||||
the same way: one unauthenticated client can spend the process's memory and time at will.
|
||||
|
||||
**Why a middleware and not `capped_body` on every route.** `routes/_common.py:capped_body` is the
|
||||
right tool where a route genuinely takes bulk bytes, and the two routes that use it use it
|
||||
correctly. But putting it on all 70 others would (a) fight the canonical "Request body → a Pydantic
|
||||
model" row, since capping means taking the body as raw bytes and parsing it by hand, and (b) leave
|
||||
the NEXT route added uncovered, because nothing would remind its author. A default that has to be
|
||||
opted into is not a default. This module inverts it: the cap applies to everything, and the few
|
||||
routes that need more opt OUT, in one list, next to the code that reads it.
|
||||
|
||||
**Why raw ASGI and not `@app.middleware("http")`.** Starlette's BaseHTTPMiddleware cannot wrap
|
||||
`receive`, so it can only read `Content-Length` — which a client may omit entirely (chunked) or
|
||||
simply understate. The bytes have to be counted as they arrive, which is the same rule `capped_body`
|
||||
already applies per-route, and counting needs the receive channel.
|
||||
|
||||
**Why the refusal is byte-identical to `capped_body`'s.** Two shapes for one fact is how the
|
||||
`cancelled`/`canceled` bug happened. The 413 body here is built through the same `JSONResponse`
|
||||
FastAPI renders `HTTPException(413, {"code": "too_large"})` with — not a hand-rolled `json.dumps`,
|
||||
whose default separators differ by two bytes — and `tests/test_body_limit.py` compares the RAW BYTES
|
||||
of the two against each other rather than against a literal, so they cannot drift apart. The same
|
||||
reasoning decides where the middleware is registered (`main.py`): just inside CORS, so this 413
|
||||
carries the same headers `capped_body`'s does.
|
||||
|
||||
**No admin knob, deliberately.** The ceiling is a constant, not a `sysconfig` value: an attacker-
|
||||
facing limit that an admin can raise from the UI is a limit that can be raised by whoever gets the
|
||||
admin session, and nobody has asked to tune it. If a self-hoster ever needs a different ceiling it
|
||||
belongs in `config.py` (env, set before the process starts), not in the database.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from app import debugsession
|
||||
from app.config import settings
|
||||
|
||||
# 1 MiB. The largest body any route on the default ceiling legitimately takes is 512 KiB (the client
|
||||
# log sink, `routes/diagnostics.py:BODY_MAX_BYTES`), so this leaves 2x headroom — a ceiling that
|
||||
# refuses a legitimate request would be a worse bug than the one this module fixes. That ratio is
|
||||
# the load-bearing fact, so `test_body_limit.py` holds it rather than the figure above.
|
||||
DEFAULT_MAX_BODY_BYTES = 1024 * 1024
|
||||
|
||||
# The routes that need MORE than the default, as (method, path TEMPLATE) → their own ceiling. The
|
||||
# template, not a concrete path, so the list reads like the route decorators it mirrors and
|
||||
# `test_body_limit.py` can assert that each one still resolves to a real route: a renamed route
|
||||
# fails the gate instead of silently losing its allowance in prod.
|
||||
#
|
||||
# These are RAISED CEILINGS, NOT EXEMPTIONS, and the difference is the whole point. An exemption
|
||||
# would mean "this route is counted by something else" — which is only true when the route's own
|
||||
# limit applies to the bytes as they ARRIVE. That, in turn, is only true of a route with NO DECLARED
|
||||
# BODY PARAMETER: FastAPI parses and spools a declared body BEFORE the first dependency runs, so a
|
||||
# route that declares one has already stored the bytes by the time its own cap — or its auth — gets
|
||||
# a say.
|
||||
#
|
||||
# Both listed routes now satisfy that. The screenshot route always did (`capped_body` streams it,
|
||||
# and the owner check resolves first). The upload route did NOT until P1: it declared
|
||||
# `file: UploadFile = File(...)`, so an anonymous caller could make the process write the whole
|
||||
# per-file allowance to temp before `require_human` refused them — this middleware bounded that
|
||||
# number but could not stop the writing. It now reads its own body from a dependency declared after
|
||||
# the auth one, so its in-handler cap measures bytes as they arrive, like everything else here.
|
||||
#
|
||||
# That rule is no longer only written here: `test_body_limit.py` parses the routes and fails the
|
||||
# gate if any route in this dict declares a body parameter. A comment could not have stopped the
|
||||
# defect it describes; the test can.
|
||||
#
|
||||
# A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why
|
||||
# `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the
|
||||
# default, so listing it could only widen what it accepts.
|
||||
ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
||||
# Ingesting a local file. The body IS the file since P1 — no multipart envelope to allow for —
|
||||
# so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s
|
||||
# `download_upload_max_bytes` is bounded by this same constant (see sysconfig.SPECS), so an
|
||||
# admin can only ever LOWER the route's cap below the edge, never raise it past one.
|
||||
#
|
||||
# WHICH LAYER ANSWERS, measured rather than assumed (deep-review round 1):
|
||||
# * knob LOWERED (an override below the default) — the route's cap is the smaller number, so
|
||||
# `capped_stream_to` trips first and answers 422 `upload_too_large` carrying the admin's own
|
||||
# figure, after draining the rest of the body so the client can read it.
|
||||
# * knob at the DEFAULT — the two numbers are EQUAL, and the edge always wins: a declared
|
||||
# Content-Length over the cap is refused here before a byte is read, and an undeclared body
|
||||
# trips this counter on the same chunk the route would have tripped on. The route's 422 is
|
||||
# unreachable in that configuration, and no amount of headroom would change it — the
|
||||
# declared-length check fires first for any body big enough to matter, and the route's drain
|
||||
# pushes the count past the edge for the rest.
|
||||
#
|
||||
# So this 413 is a refusal a real user meets on a default install, not a theoretical edge, and
|
||||
# it has to be as readable as the route's own: the client localizes `too_large` (`errors.tooLarge`)
|
||||
# exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before
|
||||
# sending it at all, off the `upload_max_bytes` that `GET /api/downloads/usage` reports.
|
||||
("POST", "/api/downloads/upload"): settings.download_upload_max_bytes,
|
||||
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
|
||||
}
|
||||
|
||||
# The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders
|
||||
# as — built through the same JSONResponse so the separators match too, and asserted against a real
|
||||
# route's raw bytes in `test_body_limit.py` rather than against a literal here.
|
||||
TOO_LARGE_BODY = JSONResponse({"detail": {"code": "too_large"}}, status_code=413).body
|
||||
|
||||
_PATH_PARAM = re.compile(r"\{[^/{}]+\}")
|
||||
|
||||
|
||||
def template_to_regex(template: str) -> re.Pattern[str]:
|
||||
"""`/api/debug-sessions/{sid}/screenshots` → a regex matching one concrete path.
|
||||
|
||||
A path parameter matches `[^/]+` — one segment, never across a `/`, so `{sid}` cannot swallow a
|
||||
deeper path and hand an exemption to a route that was never granted one.
|
||||
"""
|
||||
literals = [re.escape(part) for part in _PATH_PARAM.split(template)]
|
||||
return re.compile("^" + "[^/]+".join(literals) + "$")
|
||||
|
||||
|
||||
def _declared_length(scope: Scope) -> int | None:
|
||||
"""The `Content-Length` header as an int, or None when absent or unparseable (a chunked body has
|
||||
none, and a malformed one is not a number we may trust — both fall through to counting)."""
|
||||
for name, value in scope.get("headers", ()):
|
||||
if name == b"content-length":
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
# silent-ok: a malformed Content-Length is the caller's doing, not a fault of ours,
|
||||
# and it is about to be answered anyway — returning None drops the request into the
|
||||
# counting path, which refuses on the real bytes. Logging it would hand an anonymous
|
||||
# client a free line in our log for every request it cares to malform.
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
async def _send_too_large(send: Send) -> None:
|
||||
await send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": 413,
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"content-length", str(len(TOO_LARGE_BODY)).encode()),
|
||||
],
|
||||
}
|
||||
)
|
||||
await send({"type": "http.response.body", "body": TOO_LARGE_BODY})
|
||||
|
||||
|
||||
class BodyLimitMiddleware:
|
||||
"""Refuses any request body over its route's ceiling, before the application sees it.
|
||||
|
||||
Registered just inside CORS in `main.py` — so the 413 is decorated like every other response
|
||||
and a browser can actually read it — but outside the session, the setup gate and every auth
|
||||
dependency, none of which run for a request that is already too large.
|
||||
|
||||
Every request gets a finite ceiling: `ROUTE_MAX_BODY_BYTES` raises it for the two routes that
|
||||
genuinely take bulk bytes, and there is no path that skips counting altogether.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app: ASGIApp,
|
||||
*,
|
||||
limit: int = DEFAULT_MAX_BODY_BYTES,
|
||||
routes: dict[tuple[str, str], int] | None = None,
|
||||
) -> None:
|
||||
self.app = app
|
||||
self.limit = limit
|
||||
self._routes = tuple(
|
||||
(method, template_to_regex(path), cap)
|
||||
for (method, path), cap in (ROUTE_MAX_BODY_BYTES if routes is None else routes).items()
|
||||
)
|
||||
|
||||
def limit_for(self, method: str, path: str) -> int:
|
||||
"""This request's ceiling: the route's own if it has one, else the global default. Every
|
||||
request gets a finite number — there is no path through here that is simply not counted."""
|
||||
for m, rx, cap in self._routes:
|
||||
if m == method and rx.match(path):
|
||||
return cap
|
||||
return self.limit
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
limit = self.limit_for(scope.get("method", ""), scope.get("path", ""))
|
||||
|
||||
declared = _declared_length(scope)
|
||||
if declared is not None and declared > limit:
|
||||
# Refused on the declaration alone: nothing downstream is called, and not one byte of
|
||||
# the body is read.
|
||||
await _send_too_large(send)
|
||||
return
|
||||
|
||||
seen = 0
|
||||
refused = False
|
||||
started = False
|
||||
|
||||
async def counting_receive() -> Message:
|
||||
nonlocal seen, refused
|
||||
message = await receive()
|
||||
if message["type"] != "http.request":
|
||||
return message
|
||||
seen += len(message.get("body", b""))
|
||||
if seen > limit:
|
||||
# The app is blocked on this receive, so it has not answered yet. Telling it the
|
||||
# client went away is the only in-band way to stop it reading: `request.stream()`
|
||||
# turns an `http.disconnect` into ClientDisconnect, which unwinds the handler. Our
|
||||
# own 413 goes out below, once the app has finished unwinding.
|
||||
refused = True
|
||||
return {"type": "http.disconnect"}
|
||||
return message
|
||||
|
||||
async def guarded_send(message: Message) -> None:
|
||||
nonlocal started
|
||||
if refused and not started:
|
||||
return # discard the app's answer to the disconnect; the 413 below is the response
|
||||
started = True
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, counting_receive, guarded_send)
|
||||
except Exception: # noqa: BLE001 — only swallowed when WE caused it, see below
|
||||
# ClientDisconnect (or whatever a handler raises on a truncated read) is the expected
|
||||
# consequence of the disconnect we just faked, and the caller must get the 413 rather
|
||||
# than a 500. Anything raised when we did NOT refuse is a real failure and is re-raised
|
||||
# untouched.
|
||||
if not refused:
|
||||
raise
|
||||
|
||||
if refused and not started:
|
||||
await _send_too_large(send)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Alphabetical order in the reader's language (E30 S3).
|
||||
|
||||
Why not the database default: the Postgres image is Alpine, and musl's `en_US.utf8` collation is BYTE
|
||||
order — every accented initial sorted after "Z" ("Zorró < Életrajzi < Ötödik"), in every A→Z list the
|
||||
server orders. Postgres ships the ICU collations on the same image (dev, test, prod alike), so a text
|
||||
sort names one: Hungarian puts "Ö" after "O" and "Cs" after "Cu", English folds the accents in.
|
||||
|
||||
`collated` wraps an ORDER BY expression — and because a keyset cursor compares with the SAME expression
|
||||
(`feedquery.sort_keys`), the cursor pages in the same order it sorted in. `sorted_words` orders a Python
|
||||
list the same way through the database, so a list built in Python matches one built in SQL exactly."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import SQLColumnExpression, column, func, select, values
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.types import String
|
||||
|
||||
from app.mailtext import normalize
|
||||
|
||||
ICU: dict[str, str] = {"en": "en-x-icu", "hu": "hu-x-icu"}
|
||||
|
||||
|
||||
def collation(lang: str | None) -> str:
|
||||
"""The ICU collation for a UI language; anything unknown sorts as English."""
|
||||
return ICU[normalize(lang)]
|
||||
|
||||
|
||||
def collated(expr: SQLColumnExpression[Any], lang: str | None) -> ColumnElement[Any]:
|
||||
"""`expr` compared in the reader's alphabet. Case-folded, so "apple" and "Apple" sit together."""
|
||||
return func.lower(expr).collate(collation(lang))
|
||||
|
||||
|
||||
def sorted_words(db: Session, words: list[str], lang: str | None) -> list[str]:
|
||||
"""`words` in the reader's alphabetical order, by the same ICU collation as `collated`."""
|
||||
if not words:
|
||||
return []
|
||||
w = values(column("w", String), name="words").data([(x,) for x in words])
|
||||
return list(db.scalars(select(w.c.w).order_by(collated(w.c.w, lang), w.c.w)))
|
||||
+11
-2
@@ -1,7 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args, get_origin
|
||||
|
||||
from pydantic import ValidationInfo, field_validator, model_validator
|
||||
from pydantic import Field, ValidationInfo, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Shipped placeholder; production must override it. Used by the startup guard below.
|
||||
@@ -247,7 +247,11 @@ class Settings(BaseSettings):
|
||||
# Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker
|
||||
# finishes), an upload's size is known at ingest, so this bounds a single file directly; the
|
||||
# user's overall byte footprint is still governed by download_default_max_bytes.
|
||||
download_upload_max_bytes: int = 524_288_000 # 500 MiB
|
||||
# ge=1: this number is also the EDGE ceiling (`bodylimit.ROUTE_MAX_BODY_BYTES`), where 0 is a
|
||||
# real limit of zero rather than the route's "no per-file cap" sentinel — an instance set to 0
|
||||
# would refuse every upload, including the admin page's own way back. To lift the cap in
|
||||
# practice, set a large number.
|
||||
download_upload_max_bytes: int = Field(default=524_288_000, ge=1) # 500 MiB
|
||||
# Retention: an asset expires this many days after its last access; the GC job then
|
||||
# deletes it (once no job references it) after an additional grace window.
|
||||
download_retention_days: int = 30
|
||||
@@ -362,6 +366,11 @@ class Settings(BaseSettings):
|
||||
tmdb_enrich_interval_min: int = 120
|
||||
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||
tmdb_enrich_batch: int = 100
|
||||
# How often the people job (E30 S6) fetches a batch of person bios from TMDB and portraits through Plex,
|
||||
# in minutes, and how many of each per run: ~18k linked people at 300 per 10 min is about ten hours
|
||||
# once, then only the newcomers.
|
||||
people_enrich_interval_min: int = 10
|
||||
people_enrich_batch: int = 300
|
||||
|
||||
@property
|
||||
def allowed_email_set(self) -> set[str]:
|
||||
|
||||
+51
-2
@@ -1,6 +1,6 @@
|
||||
from collections.abc import Iterator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
from app.config import settings
|
||||
@@ -12,7 +12,9 @@ class Base(DeclarativeBase):
|
||||
|
||||
# Pool sized above the sync-endpoint threadpool (~40) so a burst of short DB sessions can't starve on
|
||||
# checkout. The image proxies deliberately DON'T hold a connection during their slow external fetch
|
||||
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones.
|
||||
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones. The one
|
||||
# long hold — a playlist sync's advisory claim, for a whole push — opens its own unpooled connection
|
||||
# (sync/playlists.py `_claim_engine`) instead of drawing a third one from here.
|
||||
engine = create_engine(
|
||||
settings.database_url,
|
||||
pool_pre_ping=True,
|
||||
@@ -34,3 +36,50 @@ def get_db() -> Iterator[Session]:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def release_read_connection(db: Session) -> None:
|
||||
"""End a request's READ transaction so its pooled connection goes back BEFORE a long await.
|
||||
|
||||
SQLAlchemy 2.0 autobegins on the first SELECT, so a request that has merely authenticated is
|
||||
already holding one of the connections above — `idle in transaction` to Postgres, and nothing
|
||||
times it out. A route that then awaits something slow (a request body still arriving, an
|
||||
external fetch) pins it for the whole wait, which is precisely the invariant the pool is sized
|
||||
on in the comment above. Measured: the next checkout after this call reuses the very backend
|
||||
that was released, and a 1.8 s upload goes from one sustained `idle in transaction` to none.
|
||||
|
||||
Every route that reads its own body from a dependency needs this, and `test_body_limit.py`
|
||||
holds that — the three that do are the ones with a raised (or self-imposed) body ceiling, and
|
||||
they got here one at a time, which is how the upload route ended up the only one fixed.
|
||||
|
||||
The caller must have NOTHING PENDING, and that is checked rather than described. The session is
|
||||
shared with every dependency that ran before, so a rollback here would silently discard a write
|
||||
one of them staged — `service.enqueue` already learned that lesson the expensive way and uses a
|
||||
SAVEPOINT for it. CLAUDE.md: a rule stated in a comment must also be enforced in code.
|
||||
|
||||
TWO checks, because the obvious one is blind to the shapes this codebase actually writes in.
|
||||
`db.new`/`deleted`/`dirty` hold only UNFLUSHED unit-of-work: the moment a helper calls
|
||||
`db.flush()` — CLAUDE.md's canonical split is "the handler commits; helpers flush" — the object
|
||||
is persistent with no unflushed history and all three sets are empty again, and Core DML
|
||||
(`db.execute(update(...))`) never enters them at all. Measured: after `add`+`flush` the sets are
|
||||
empty, the old check passed, and the rollback lost the row. So the transaction is asked
|
||||
directly. Postgres assigns a transaction id on the first WRITE and never for a read, so
|
||||
`pg_current_xact_id_if_assigned()` is NULL for exactly the state this function is safe in — and
|
||||
it sees a raw `text("UPDATE …")` too, which no session-level bookkeeping would.
|
||||
"""
|
||||
if db.new or db.deleted or db.dirty:
|
||||
raise RuntimeError(
|
||||
"release_read_connection() with pending work: something ahead of this dependency "
|
||||
"staged a write, which ending the transaction would discard. Commit it first, or do "
|
||||
"it after the await."
|
||||
)
|
||||
if (
|
||||
db.in_transaction()
|
||||
and db.execute(text("SELECT pg_current_xact_id_if_assigned()")).scalar() is not None
|
||||
):
|
||||
raise RuntimeError(
|
||||
"release_read_connection() on a transaction that has already WRITTEN: something ahead "
|
||||
"of this dependency flushed a row or ran an UPDATE/INSERT/DELETE, which ending the "
|
||||
"transaction would discard. Commit it first, or do it after the await."
|
||||
)
|
||||
db.rollback()
|
||||
|
||||
@@ -48,6 +48,7 @@ def pick(
|
||||
shelf: list[dict] | None = None,
|
||||
exclude: set[str] | None = None,
|
||||
start: int = 0,
|
||||
lang: str,
|
||||
) -> dict | None:
|
||||
"""`{"tier": ..., "card": ...}`, or None when nothing at all fits.
|
||||
|
||||
@@ -70,8 +71,8 @@ def pick(
|
||||
skip = exclude or set()
|
||||
|
||||
def _resume() -> dict | None:
|
||||
for card in rails.continue_watching(db, user) if resume is None else resume:
|
||||
playable = rails.playable(db, user.id, card)
|
||||
for card in rails.continue_watching(db, user, lang=lang) if resume is None else resume:
|
||||
playable = rails.playable(db, user.id, card, lang=lang)
|
||||
# Excluded on the card the USER saw (the show), not only on what it resolves to, so a
|
||||
# rerolled-past show does not come straight back as its own next episode.
|
||||
if cards.key(card) in skip or (playable is not None and cards.key(playable) in skip):
|
||||
@@ -82,7 +83,7 @@ def pick(
|
||||
|
||||
def _comfort() -> dict | None:
|
||||
if shelf is None:
|
||||
own = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
||||
own = pins.shelf(db, user, include_plex=rails.plex_available(db), lang=lang)
|
||||
else:
|
||||
own = shelf
|
||||
for card in pins.playables(own):
|
||||
@@ -93,7 +94,7 @@ def pick(
|
||||
def _surprise() -> dict | None:
|
||||
# A wider draw than the one card we need: the shuffle is seeded, so without the headroom an
|
||||
# excluded pick would leave the tier empty rather than move past it.
|
||||
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW):
|
||||
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW, lang=lang):
|
||||
if cards.key(card) not in skip:
|
||||
return card
|
||||
return None
|
||||
|
||||
@@ -170,7 +170,7 @@ class _Batch:
|
||||
return b
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict | None:
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch, lang: str) -> dict | None:
|
||||
"""One pin as a unified card with its resolved `next`. None when the pinned row has gone (the FK
|
||||
cascade removes the pin itself, so this is only the window between a delete and a refetch)."""
|
||||
card: dict | None = None
|
||||
@@ -196,7 +196,7 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict
|
||||
if it is None:
|
||||
return None
|
||||
st = rollup.item_state_map(db, user.id, [it.id]).get(it.id)
|
||||
card = cards.from_plex(leaf_card(it, st))
|
||||
card = cards.from_plex(leaf_card(it, st, lang=lang))
|
||||
nxt = dict(card) # a movie or a single episode plays itself
|
||||
|
||||
elif pin.plex_show_id is not None:
|
||||
@@ -204,8 +204,8 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict
|
||||
if sh is None:
|
||||
return None
|
||||
status = rollup.state_map(db, user.id, [sh.id]).get(sh.id, "new")
|
||||
card = cards.from_plex(show_card(sh, status))
|
||||
ep = queries.next_episode(db, user.id, sh.id)
|
||||
card = cards.from_plex(show_card(sh, status, lang=lang))
|
||||
ep = queries.next_episode(db, user.id, sh.id, lang=lang)
|
||||
nxt = cards.from_plex(ep) if ep else None
|
||||
|
||||
elif pin.playlist_id is not None:
|
||||
@@ -229,7 +229,7 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict
|
||||
_PLEX_COLUMNS = ("plex_item_id", "plex_show_id")
|
||||
|
||||
|
||||
def shelf(db: Session, user: User, *, include_plex: bool = True) -> list[dict]:
|
||||
def shelf(db: Session, user: User, *, include_plex: bool = True, lang: str) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order.
|
||||
|
||||
`include_plex=False` is the Home surface with the deployment's Plex toggle off (the owner's call,
|
||||
@@ -243,7 +243,7 @@ def shelf(db: Session, user: User, *, include_plex: bool = True) -> list[dict]:
|
||||
pins = [p for p in pins if all(getattr(p, c) is None for c in _PLEX_COLUMNS)]
|
||||
batch = _Batch.load(db, user, pins)
|
||||
for pin in pins:
|
||||
card = _shelf_card(db, user, pin, batch)
|
||||
card = _shelf_card(db, user, pin, batch, lang)
|
||||
if card is not None:
|
||||
out.append(card)
|
||||
return out
|
||||
|
||||
@@ -40,7 +40,7 @@ from app.models import (
|
||||
VideoState,
|
||||
)
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, movie_card, show_card
|
||||
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||
from app.plex.query import apply_show_state_filter, content_libs
|
||||
|
||||
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
|
||||
@@ -221,7 +221,9 @@ def _unseen_movie_conds(uid: int):
|
||||
return ~seen
|
||||
|
||||
|
||||
def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tuple[object, dict]]:
|
||||
def plex_in_progress(
|
||||
db: Session, uid: int, limit: int = RAIL_LIMIT, *, lang: str
|
||||
) -> list[tuple[object, dict]]:
|
||||
"""In-progress movies + partially-watched shows as (progressed_at, card).
|
||||
|
||||
Mirrors `routes/plex/discover.py::_continue_watching` — same `PlexState` predicate for movies (a
|
||||
@@ -249,7 +251,7 @@ def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tup
|
||||
.limit(limit)
|
||||
)
|
||||
for it, st in db.execute(mq).all():
|
||||
out.append((st.progress_updated_at, movie_card(it, st)))
|
||||
out.append((st.progress_updated_at, movie_card(it, st, lang=lang)))
|
||||
|
||||
if show_libs:
|
||||
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
||||
@@ -263,12 +265,14 @@ def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tup
|
||||
for sh, resumed_at in db.execute(
|
||||
sq.order_by(prog.c.resumed_at.desc().nullslast()).limit(limit)
|
||||
).all():
|
||||
out.append((resumed_at, show_card(sh, "in_progress")))
|
||||
out.append((resumed_at, show_card(sh, "in_progress", lang=lang)))
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) -> list[dict]:
|
||||
def plex_surprise(
|
||||
db: Session, uid: int, budget: str, seed: int, limit: int, *, lang: str
|
||||
) -> list[dict]:
|
||||
"""Unseen playable movies inside the budget, in seeded-shuffle order.
|
||||
|
||||
Movies only. A show is not a thing you can start playing, and resolving every candidate show to
|
||||
@@ -300,14 +304,14 @@ def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) ->
|
||||
q = q.where(PlexItem.duration_s <= hi)
|
||||
items = db.execute(q).scalars().all()
|
||||
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||
return [movie_card(it, states.get(it.id)) for it in items]
|
||||
return [movie_card(it, states.get(it.id), lang=lang) for it in items]
|
||||
|
||||
|
||||
# Watch states that take an episode out of the running for "what would play next".
|
||||
_DONE = ("watched", "hidden")
|
||||
|
||||
|
||||
def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
||||
def next_episode(db: Session, uid: int, show_id: int, *, lang: str) -> dict | None:
|
||||
"""What "put that show on" means: the episode in progress, else the first unwatched one, else the
|
||||
first — over the same `(season, episode)` broadcast ordering `detail.py::_episode_nav` uses.
|
||||
|
||||
@@ -357,5 +361,9 @@ def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
||||
ep, st = pick
|
||||
show = db.get(PlexShow, show_id)
|
||||
return leaf_card(
|
||||
ep, st, show_title=show.title if show else None, show_rk=show.rating_key if show else None
|
||||
ep,
|
||||
st,
|
||||
show_title=display_title(show, lang) if show else None,
|
||||
show_rk=show.rating_key if show else None,
|
||||
lang=lang,
|
||||
)
|
||||
|
||||
@@ -22,7 +22,7 @@ def plex_available(db: Session) -> bool:
|
||||
return sysconfig.get_bool(db, "plex_enabled")
|
||||
|
||||
|
||||
def playable(db: Session, uid: int, card: dict) -> dict | None:
|
||||
def playable(db: Session, uid: int, card: dict, *, lang: str) -> dict | None:
|
||||
"""The thing that would actually START from this card — itself for anything with a runtime, and
|
||||
for a Plex SHOW the episode that is next (in progress, else the first unwatched, else the first).
|
||||
|
||||
@@ -40,11 +40,11 @@ def playable(db: Session, uid: int, card: dict) -> dict | None:
|
||||
sh = by_rating_key(db, PlexShow, card["id"])
|
||||
if sh is None:
|
||||
return None
|
||||
ep = queries.next_episode(db, uid, sh.id)
|
||||
ep = queries.next_episode(db, uid, sh.id, lang=lang)
|
||||
return cards.from_plex(ep) if ep else None
|
||||
|
||||
|
||||
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[dict]:
|
||||
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT, *, lang: str) -> list[dict]:
|
||||
"""One resume rail across both sources, most recently progressed first.
|
||||
|
||||
A Plex SHOW enters as a show CARD — that is what the Plex rail shows, and it is the artwork the
|
||||
@@ -57,17 +57,22 @@ def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[
|
||||
(ts, cards.from_video(r)) for ts, r in queries.yt_in_progress(db, user, limit)
|
||||
]
|
||||
if plex_available(db):
|
||||
rows += [(ts, cards.from_plex(c)) for ts, c in queries.plex_in_progress(db, user.id, limit)]
|
||||
rows += [
|
||||
(ts, cards.from_plex(c))
|
||||
for ts, c in queries.plex_in_progress(db, user.id, limit, lang=lang)
|
||||
]
|
||||
rows.sort(key=lambda r: r[0] or NO_TS, reverse=True)
|
||||
out = [card for _, card in rows[:limit]]
|
||||
# Resolve AFTER the slice: only the rows that survive it are worth a query.
|
||||
for card in out:
|
||||
if card["kind"] == "show":
|
||||
card["next"] = playable(db, user.id, card)
|
||||
card["next"] = playable(db, user.id, card, lang=lang)
|
||||
return out
|
||||
|
||||
|
||||
def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) -> list[dict]:
|
||||
def surprise(
|
||||
db: Session, user: User, budget: str, seed: int, limit: int = 3, *, lang: str
|
||||
) -> list[dict]:
|
||||
"""The hero's draw pool: unseen things that fit the budget, from both sources under ONE seed.
|
||||
|
||||
Interleaved rather than concatenated — the same alternation `routes/plex/discover.py` uses for its
|
||||
@@ -83,7 +88,9 @@ def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) ->
|
||||
(cards.from_video, queries.yt_surprise(db, user, budget, seed, limit)),
|
||||
(
|
||||
cards.from_plex,
|
||||
queries.plex_surprise(db, user.id, budget, seed, limit) if plex_available(db) else [],
|
||||
queries.plex_surprise(db, user.id, budget, seed, limit, lang=lang)
|
||||
if plex_available(db)
|
||||
else [],
|
||||
),
|
||||
]
|
||||
if seed % 2:
|
||||
|
||||
@@ -25,7 +25,7 @@ from app.downloads.jobdb import (
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
|
||||
@@ -597,7 +597,7 @@ def _download(
|
||||
}
|
||||
if job and not job.display_name and meta.title:
|
||||
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
||||
fields["display_name"] = meta.title[:255]
|
||||
fields["display_name"] = meta.title[:DISPLAY_TEXT_MAX]
|
||||
# Atomic terminal write: finalizes ONLY while the row is still running, so a child that
|
||||
# finishes just after a route cancel/pause can't resurrect it (see finalize_if_running).
|
||||
finalized = service.finalize_if_running(db, job_id, **fields)
|
||||
|
||||
@@ -20,7 +20,7 @@ from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobStatus
|
||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||
from app.models import DISPLAY_TEXT_MAX, Channel, DownloadJob, MediaAsset, Video
|
||||
|
||||
|
||||
class EditError(Exception):
|
||||
@@ -179,7 +179,7 @@ def enqueue(
|
||||
profile_id=profile_id,
|
||||
profile_snapshot=spec,
|
||||
display_name=display_name
|
||||
or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
||||
or (asset.title[:DISPLAY_TEXT_MAX] if asset.status == "ready" and asset.title else None),
|
||||
status="queued",
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
@@ -198,10 +198,13 @@ def enqueue(
|
||||
|
||||
|
||||
def _clip_title(source_title: str | None, display_name: str | None) -> str:
|
||||
# Clipped to DISPLAY_TEXT_MAX, not to a literal: this feeds `DownloadJob.display_name` through
|
||||
# `enqueue_edit`, the same column `display_text` guards. A machine-made title, so it is clipped
|
||||
# rather than run through `display_text` — blank here means "no title", not None.
|
||||
if display_name:
|
||||
return display_name[:255]
|
||||
return display_name[:DISPLAY_TEXT_MAX]
|
||||
base = source_title or "Clip"
|
||||
return f"{base} (clip)"[:255]
|
||||
return f"{base} (clip)"[:DISPLAY_TEXT_MAX]
|
||||
|
||||
|
||||
def enqueue_edit(
|
||||
@@ -255,7 +258,7 @@ def enqueue_edit(
|
||||
source_asset_id=src_asset.id,
|
||||
edit_spec=spec,
|
||||
profile_snapshot={},
|
||||
display_name=display_name or asset.title,
|
||||
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
|
||||
status="queued",
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
|
||||
@@ -25,7 +25,7 @@ from sqlalchemy.orm import Session
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, probe, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.models import DownloadJob, MediaAsset, display_text
|
||||
|
||||
|
||||
def _title_from_filename(filename: str) -> str:
|
||||
@@ -99,7 +99,7 @@ def ingest_upload(
|
||||
source_kind="upload",
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
display_name=(display_name or "").strip() or None,
|
||||
display_name=display_text(display_name),
|
||||
status="done",
|
||||
progress=100,
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
|
||||
@@ -15,6 +15,7 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.collation import collated
|
||||
from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
@@ -478,8 +479,10 @@ def feed_years(db: Session, user: User, **params) -> list[dict]:
|
||||
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
|
||||
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
||||
# (OFFSET would shift and duplicate/skip rows).
|
||||
def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) -> list[dict]:
|
||||
"""`lang` orders the title sorts in the reader's alphabet (`collation`); the other keys ignore it."""
|
||||
pub = {"expr": Video.published_at, "asc": False, "nullable": False, "kind": "dt"}
|
||||
title = collated(Video.title, lang)
|
||||
table: dict[str, list[dict]] = {
|
||||
"newest": [pub],
|
||||
"oldest": [{**pub, "asc": True}],
|
||||
@@ -491,10 +494,8 @@ def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
"duration_asc": [
|
||||
{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}
|
||||
],
|
||||
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
|
||||
"title_desc": [
|
||||
{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}
|
||||
],
|
||||
"title": [{"expr": title, "asc": True, "nullable": True, "kind": "str"}],
|
||||
"title_desc": [{"expr": title, "asc": False, "nullable": True, "kind": "str"}],
|
||||
"subscribers": [
|
||||
{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}
|
||||
],
|
||||
|
||||
@@ -3,7 +3,7 @@ before a line of this was written (plans/e16-debug-issue.md, S0).
|
||||
|
||||
Configured by an admin (sysconfig `issues_gitea_url` / `_repo` / `_token`, the token encrypted at
|
||||
rest). The repository must be PRIVATE: an issue carries its reporter's log — page titles, filter
|
||||
values, at Teljes what they typed into search — and a public repository would publish it. That is
|
||||
values, at Teljes (TRACE) what they typed into search — and a public repository would publish it. That is
|
||||
checked on every send, not only in the admin's "test connection", because a repository can be made
|
||||
public after the fact.
|
||||
|
||||
@@ -154,9 +154,22 @@ def check(db: Session) -> dict:
|
||||
|
||||
|
||||
def preview(db: Session, text: str) -> str:
|
||||
"""The body as Gitea itself will render it (its markdown flavour, its sanitiser)."""
|
||||
with _client(_settings(db)) as (client, repo):
|
||||
r = _call(client, "POST", "/markdown", json={"Text": text, "Mode": "gfm", "Context": repo})
|
||||
"""The body as Gitea itself will render it (its markdown flavour, its sanitiser).
|
||||
|
||||
**No `Context`, deliberately — a repository context here does not fail the call, it kills the
|
||||
tracker.** With one, Gitea's post-processor turns every hex-shaped word into a commit link, and
|
||||
on the API's render path the commit checker it asks is nil: the whole Gitea process segfaults
|
||||
and restarts, taking the git server (and, here, the container registry) down with it. Every
|
||||
report carries hex — a tab id, a request id — so this was not an edge case but the only case.
|
||||
Measured on throwaway instances (2026-09-20): `{"Mode": "gfm", "Context": "<repo>"}` with
|
||||
`5645b1c0ae01` in the text crashes 1.26.4 and 1.27.1 and is fixed in 1.27.3 — but a self-hoster
|
||||
runs whatever version they run, so the payload stays safe here rather than relying on theirs.
|
||||
Dropping `Context` is not a downgrade of the preview: measured against the same body, the HTML
|
||||
keeps its tables, its task lists and its highlighted code blocks. It only stops resolving
|
||||
repo-relative links (a hash, a `#123`), which the issue page itself still resolves.
|
||||
"""
|
||||
with _client(_settings(db)) as (client, _repo):
|
||||
r = _call(client, "POST", "/markdown", json={"Text": text, "Mode": "gfm"})
|
||||
return r.text
|
||||
|
||||
|
||||
|
||||
@@ -466,7 +466,6 @@ _THIRD_PARTY_LEVELS = {
|
||||
"httpx": "WARNING",
|
||||
"httpcore": "WARNING",
|
||||
"asyncio": "WARNING",
|
||||
"multipart": "WARNING",
|
||||
"PIL": "WARNING",
|
||||
"alembic": "INFO",
|
||||
# uvicorn's own startup/shutdown/WebSocket lines are worth keeping; its ACCESS log is not —
|
||||
|
||||
+31
-10
@@ -23,6 +23,7 @@ from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.exc import DataError
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from app import bodylimit
|
||||
from app import logsink
|
||||
from app import state
|
||||
from app import sysconfig
|
||||
@@ -189,15 +190,6 @@ app.add_middleware(
|
||||
https_only=settings.session_https_only, # Secure flag when served over HTTPS (prod)
|
||||
)
|
||||
|
||||
if settings.frontend_origin:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[settings.frontend_origin],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Paths reachable while the instance is unconfigured (setup mode). Everything else under /api or
|
||||
# /auth is locked until the wizard finishes; static/SPA always loads so the wizard page can render.
|
||||
_SETUP_OPEN_PREFIXES = ("/api/setup", "/api/version")
|
||||
@@ -217,6 +209,34 @@ async def setup_gate(request, call_next):
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# SEC-20 — the global request-body cap (app/bodylimit.py). FastAPI reads a declared body IN FULL
|
||||
# before the first dependency, so without this the anonymous login route is a free memory/time sink.
|
||||
#
|
||||
# WHERE it sits is the whole design, and it is pinned by `test_body_limit.py` rather than by this
|
||||
# comment, because `add_middleware` inverts the reading order (last registered = OUTERMOST) and a
|
||||
# stack described in prose drifts. Registered here it lands directly INSIDE CORS and directly
|
||||
# OUTSIDE the setup gate, which is the narrowest position that satisfies both constraints:
|
||||
#
|
||||
# * inside CORS and the security headers, so the 413 is decorated like every other response. A
|
||||
# refusal the browser drops for a missing Access-Control-Allow-Origin is no better than a hang,
|
||||
# and `capped_body`'s 413 one layer down IS decorated — two shapes for one fact is exactly what
|
||||
# bodylimit.py exists to avoid.
|
||||
# * outside the setup gate, the session and every auth dependency under the router — none of which
|
||||
# run for a body that is already too large. The setup gate matters specifically: on an instance
|
||||
# that is not yet configured it opens a DB session per request, which is precisely the anonymous
|
||||
# pre-auth work SEC-20 exists to remove.
|
||||
app.add_middleware(bodylimit.BodyLimitMiddleware)
|
||||
|
||||
if settings.frontend_origin:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[settings.frontend_origin],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def static_cache_headers(request, call_next):
|
||||
"""Long-cache the content-hashed SPA bundles. Vite hashes their filename, so a given
|
||||
@@ -259,7 +279,8 @@ async def security_headers(request, call_next):
|
||||
|
||||
# The access log (app/logs.py): request id, route template, status, duration — one line, never a
|
||||
# query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole
|
||||
# request, including what the three above do. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# request, including what every layer above does — the body cap included, so a 413 is still logged
|
||||
# with its request id and duration. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# which is why the account comes from `request.state.user_id` (set by the auth dependency, the one
|
||||
# place a session is actually resolved into an account) rather than from `request.session`.
|
||||
app.middleware("http")(log_requests)
|
||||
|
||||
+41
-6
@@ -23,6 +23,34 @@ from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
|
||||
from app.db import Base
|
||||
|
||||
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
||||
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
||||
# `User.display_name` (their Google profile name), `DownloadJob.display_name` and
|
||||
# `DownloadJob.display_uploader`. A value wider than the column is a psycopg DataError, which
|
||||
# `main.py` re-raises for everything but the NUL case — i.e. a bare 500 for a name somebody pasted.
|
||||
DISPLAY_TEXT_MAX = 255
|
||||
|
||||
|
||||
def display_text(raw: str | None) -> str | None:
|
||||
"""A user-supplied display string on its way into one of those columns: stripped, truncated,
|
||||
blank -> None.
|
||||
|
||||
One definition because there are SIX writers and they had drifted: `POST /api/downloads`,
|
||||
`POST /api/downloads/edit`, `POST /api/downloads/upload`, the metadata PATCH's two fields, and
|
||||
the Google sign-in callback. Only the PATCH truncated. Truncating is the right answer rather
|
||||
than refusing: the name is a label on work the user has already done — a file already
|
||||
transferred, an account already authenticated — and losing its tail is not worth failing the
|
||||
request over.
|
||||
|
||||
`test_upload.py` holds both halves: that every route assigning one of these fields goes through
|
||||
this helper (parsed, not grepped), and that each of those routes truncates rather than 500s. A
|
||||
sweep is exactly the kind of thing that gets done five-sixths of the way, which is how this
|
||||
ended up being written twice.
|
||||
"""
|
||||
return (raw or "").strip()[:DISPLAY_TEXT_MAX] or None
|
||||
|
||||
|
||||
from app.utils import canonical_email
|
||||
|
||||
|
||||
@@ -114,7 +142,7 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
# not cleared eagerly — it simply stops counting).
|
||||
diagnostics_level: Mapped[str | None] = mapped_column(String(8))
|
||||
diagnostics_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
avatar_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
role: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
# The user's own YouTube channel id, resolved lazily (channels.list?mine=true) and
|
||||
@@ -953,11 +981,11 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||
)
|
||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
||||
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
||||
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
extra_links: Mapped[list | None] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(
|
||||
@@ -1196,6 +1224,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
||||
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline (E30)
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
@@ -1357,6 +1386,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
||||
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline (E30)
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
@@ -1387,7 +1417,8 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
||||
top few cast. Rows whose name no longer appears in ANY item's cast_names/directors (a library
|
||||
removal orphaned them) are pruned by the maintenance sweep, mirroring how Plex drops orphaned
|
||||
metadata. The filterable cast lives in plex_items/plex_shows.cast_names (self-cleaning with the
|
||||
row); this table is ONLY the headshot cache."""
|
||||
row); this table is the people's own facts: the headshot link, the TMDB link and the TMDB bio
|
||||
(the headshot FILE lives in the image cache, filled by the people_enrich job)."""
|
||||
|
||||
__tablename__ = "plex_people"
|
||||
|
||||
@@ -1397,11 +1428,15 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# TMDB person id, linked by name from a title's TMDB credits (E5 S2 lays the foundation; the /person
|
||||
# biography fetch + person-page render land in S6). NULL until a TMDB-enriched title names this person.
|
||||
tmdb_person_id: Mapped[int | None] = mapped_column(Integer)
|
||||
# --- TMDB person enrichment (E5 S6): fetched from /person/{tmdb_person_id} the first time the person
|
||||
# page is opened, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||
# --- TMDB person enrichment (E5 S6): fetched from /person/{tmdb_person_id} by the people_enrich job
|
||||
# (E30 S6), or by the first person-page open the job hasn't reached, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||
# no-op fetch so it isn't retried every open). `known_for` = top combined-credits [{tmdb_id,
|
||||
# media_type, title}], resolved to library rows by guid at serve time (like recommendations). ---
|
||||
biography: Mapped[str | None] = mapped_column(Text)
|
||||
# TMDB's Hungarian biography (E30 S4), from the same request's `translations`; a Hungarian reader is
|
||||
# served it, and the instance-language `biography` when TMDB has none. Birthplace, aliases and
|
||||
# keywords have no Hungarian on TMDB, so they stay in the instance language.
|
||||
biography_hu: Mapped[str | None] = mapped_column(Text)
|
||||
birthday: Mapped[str | None] = mapped_column(String(10)) # "YYYY-MM-DD", served as-is
|
||||
deathday: Mapped[str | None] = mapped_column(String(10))
|
||||
place_of_birth: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
+32
-14
@@ -15,20 +15,35 @@ def _art_url(rating_key: str, art_key: str | None) -> str | None:
|
||||
|
||||
|
||||
# Card hover-overview: a short synopsis for the reveal panel, trimmed so a strip/related row of cards
|
||||
# stays a few hundred bytes each rather than carrying full synopses. (These non-grid card paths serve
|
||||
# the EN summary/rating; the main grid coalesces the HU twin in browse.unified_library.)
|
||||
# stays a few hundred bytes each rather than carrying full synopses.
|
||||
def _card_summary(summary: str | None) -> str | None:
|
||||
return (summary or "")[:300] or None
|
||||
|
||||
|
||||
def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
||||
def norm_lang(lang: str | None) -> str:
|
||||
"""The overlay has an EN base + an optional HU twin; anything other than `hu` serves EN."""
|
||||
return "hu" if lang == "hu" else "en"
|
||||
|
||||
|
||||
# Every card speaks the reader's language (E30): `lang="hu"` serves the HU overlay twin's title and
|
||||
# summary, falling back to EN where a row has none — the same rule as browse.unified_library's
|
||||
# `title_col`/`summary_col`. These rails once served English to a Hungarian reader.
|
||||
def display_title(row: PlexItem | PlexShow, lang: str) -> str:
|
||||
return (row.title_hu or row.title) if lang == "hu" else row.title
|
||||
|
||||
|
||||
def display_summary(row: PlexItem | PlexShow, lang: str) -> str | None:
|
||||
return (row.summary_hu or row.summary) if lang == "hu" else row.summary
|
||||
|
||||
|
||||
def movie_card(it: PlexItem, st: PlexState | None, *, lang: str) -> dict:
|
||||
return {
|
||||
"id": it.rating_key,
|
||||
"type": "movie",
|
||||
"title": it.title,
|
||||
"title": display_title(it, lang),
|
||||
"year": it.year,
|
||||
"rating": it.rating,
|
||||
"summary": _card_summary(it.summary),
|
||||
"summary": _card_summary(display_summary(it, lang)),
|
||||
"duration_seconds": it.duration_s,
|
||||
"thumb": f"/api/plex/image/{it.rating_key}",
|
||||
"art": _art_url(it.rating_key, it.art_key),
|
||||
@@ -39,14 +54,14 @@ def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def show_card(sh: PlexShow, status: str = "new") -> dict:
|
||||
def show_card(sh: PlexShow, status: str = "new", *, lang: str) -> dict:
|
||||
return {
|
||||
"id": sh.rating_key,
|
||||
"type": "show",
|
||||
"title": sh.title,
|
||||
"title": display_title(sh, lang),
|
||||
"year": sh.year,
|
||||
"rating": sh.rating,
|
||||
"summary": _card_summary(sh.summary),
|
||||
"summary": _card_summary(display_summary(sh, lang)),
|
||||
"thumb": f"/api/plex/image/{sh.rating_key}",
|
||||
"art": _art_url(sh.rating_key, sh.art_key),
|
||||
"season_count": sh.child_count,
|
||||
@@ -55,12 +70,12 @@ def show_card(sh: PlexShow, status: str = "new") -> dict:
|
||||
}
|
||||
|
||||
|
||||
def episode_card(e: PlexItem, st: PlexState | None) -> dict:
|
||||
def episode_card(e: PlexItem, st: PlexState | None, *, lang: str) -> dict:
|
||||
return {
|
||||
"id": e.rating_key,
|
||||
"type": "episode",
|
||||
"title": e.title,
|
||||
"summary": e.summary,
|
||||
"title": display_title(e, lang),
|
||||
"summary": display_summary(e, lang),
|
||||
"season_number": e.season_number,
|
||||
"episode_number": e.episode_number,
|
||||
"duration_seconds": e.duration_s,
|
||||
@@ -77,12 +92,15 @@ def leaf_card(
|
||||
st: PlexState | None,
|
||||
show_title: str | None = None,
|
||||
show_rk: str | None = None,
|
||||
*,
|
||||
lang: str,
|
||||
) -> dict:
|
||||
"""A playable-leaf card (movie or episode). Used by playlists, which can mix both kinds.
|
||||
`show_rk` (the show's rating_key) lets the client group a playlist's episodes by show/season."""
|
||||
`show_rk` (the show's rating_key) lets the client group a playlist's episodes by show/season.
|
||||
`show_title` is the caller's, already in `lang` (see `display_title`)."""
|
||||
if it.kind == "episode":
|
||||
card = episode_card(it, st)
|
||||
card = episode_card(it, st, lang=lang)
|
||||
card["show_title"] = show_title
|
||||
card["show_id"] = show_rk
|
||||
return card
|
||||
return movie_card(it, st)
|
||||
return movie_card(it, st, lang=lang)
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""One Hungarian word per Plex genre (E30).
|
||||
|
||||
Plex's Hungarian agent returns a MIXED vocabulary: some genres come back translated ("Dráma",
|
||||
"Vígjáték"), others stay English ("Action", "Adventure", "Biography"), and a title with no HU twin has
|
||||
only its English genres. The HU genre list therefore offered "Drama" beside "Dráma" and a dozen English
|
||||
words. This map sends every genre word we have met — English (Plex EN agent, TMDB, IMDb) or Plex's
|
||||
Hungarian — to the ONE word a Hungarian reader sees. The overlay sync writes it into `genres_hu`, and a
|
||||
genre filter is read through it too, so a saved filter still holding "Action" keeps matching "Akció".
|
||||
|
||||
A word not in the map passes through unchanged: an unknown genre stays visible (in whatever language
|
||||
Plex gave it) rather than disappearing.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
# Keys are matched case-insensitively (see `_BY_KEY`). The Hungarian values double as keys mapping to
|
||||
# themselves, so running a list through twice changes nothing.
|
||||
_HU: dict[str, str] = {
|
||||
"Action": "Akció",
|
||||
"Action & Adventure": "Akció és kaland",
|
||||
"Adventure": "Kaland",
|
||||
"Animation": "Animációs",
|
||||
"Anime": "Anime",
|
||||
"Biography": "Életrajzi",
|
||||
"Children": "Gyerek",
|
||||
"Comedy": "Vígjáték",
|
||||
"Crime": "Bűnügyi",
|
||||
"Documentary": "Dokumentum",
|
||||
"Drama": "Dráma",
|
||||
"Family": "Családi",
|
||||
"Fantasy": "Fantasy",
|
||||
"Film-Noir": "Film noir",
|
||||
"Food": "Gasztronómia",
|
||||
"Game Show": "Vetélkedő",
|
||||
"Game-Show": "Vetélkedő",
|
||||
"History": "Történelmi",
|
||||
"Home and Garden": "Otthon és kert",
|
||||
"Horror": "Horror",
|
||||
"Indie": "Független",
|
||||
"Kids": "Gyerek",
|
||||
"Martial Arts": "Harcművészet",
|
||||
"Mini-Series": "Minisorozat",
|
||||
"Music": "Zenei",
|
||||
"Musical": "Musical",
|
||||
"Mystery": "Rejtély",
|
||||
"News": "Hírek",
|
||||
"Reality": "Valóságshow",
|
||||
"Reality-TV": "Valóságshow",
|
||||
"Romance": "Romantikus",
|
||||
"Sci-Fi": "Sci-Fi",
|
||||
"Sci-Fi & Fantasy": "Sci-Fi és fantasy",
|
||||
"Science Fiction": "Sci-Fi",
|
||||
"Short": "Rövidfilm",
|
||||
"Soap": "Szappanopera",
|
||||
"Sport": "Sport",
|
||||
"Suspense": "Feszültség",
|
||||
"Talk": "Talkshow",
|
||||
"Talk Show": "Talkshow",
|
||||
"Thriller": "Thriller",
|
||||
"Travel": "Utazás",
|
||||
"TV Movie": "TV film",
|
||||
"War": "Háborús",
|
||||
"War & Politics": "Háború és politika",
|
||||
"Western": "Western",
|
||||
# Plex's own Hungarian words, pinned to themselves (and to the spelling above where they differ).
|
||||
"Akció": "Akció",
|
||||
"Animációs": "Animációs",
|
||||
"Bűnügyi": "Bűnügyi",
|
||||
"Családi": "Családi",
|
||||
"Dokumentum": "Dokumentum",
|
||||
"Dokumentumfilm": "Dokumentum",
|
||||
"Dráma": "Dráma",
|
||||
"Életrajzi": "Életrajzi",
|
||||
"Háborús": "Háborús",
|
||||
"Kaland": "Kaland",
|
||||
"Rejtély": "Rejtély",
|
||||
"Romantikus": "Romantikus",
|
||||
"Történelmi": "Történelmi",
|
||||
"TV film": "TV film",
|
||||
"Vígjáték": "Vígjáték",
|
||||
"Zene": "Zenei",
|
||||
"Zenei": "Zenei",
|
||||
}
|
||||
_BY_KEY = {k.casefold(): v for k, v in _HU.items()}
|
||||
|
||||
|
||||
def hu_genre(word: str) -> str:
|
||||
"""The Hungarian word for one genre; an unknown word comes back unchanged."""
|
||||
return _BY_KEY.get(word.strip().casefold(), word)
|
||||
|
||||
|
||||
def hu_genres(words: Iterable[str] | None) -> list[str] | None:
|
||||
"""A genre list in Hungarian, in its first-seen order, each word once ("Drama" + "Dráma" → one
|
||||
"Dráma"). None or empty stays None, like the listing's own `_tags`."""
|
||||
out: list[str] = []
|
||||
for w in words or ():
|
||||
hu = hu_genre(w)
|
||||
if hu not in out:
|
||||
out.append(hu)
|
||||
return out or None
|
||||
@@ -8,6 +8,9 @@ few hundred MB at most for a big library — acceptable, matching the download d
|
||||
The cache directory (``cache.CACHE_DIR``) is shared with the subtitle cache (see ``subtitles.py``).
|
||||
"""
|
||||
|
||||
import time
|
||||
from datetime import timedelta
|
||||
|
||||
from fastapi import Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -56,6 +59,49 @@ def read(key: str) -> tuple[bytes, str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def has(key: str) -> bool:
|
||||
"""A cached, non-empty file exists for `key` (a prefetch's cheap check — no read)."""
|
||||
for ext, _ in _EXT_CT:
|
||||
try:
|
||||
if (CACHE_DIR / f"{key}.{ext}").stat().st_size > 0:
|
||||
return True
|
||||
# silent-ok: a miss, as in read().
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def drop(key: str) -> None:
|
||||
"""Remove `key`'s file, whatever its extension (a superseded variant). Best-effort."""
|
||||
for ext, _ in _EXT_CT:
|
||||
try:
|
||||
(CACHE_DIR / f"{key}.{ext}").unlink(missing_ok=True)
|
||||
# silent-ok: a leftover original only costs disk; the small twin is already written.
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def mark_miss(key: str) -> None:
|
||||
"""Remember that `key`'s source could not be fetched (an empty `.miss` file; its mtime is when), so
|
||||
a prefetch can leave a dead source alone for a while. Best-effort."""
|
||||
try:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(CACHE_DIR / f"{key}.miss").touch()
|
||||
# silent-ok: without the mark the dead source is only asked for again on the next run.
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def missed_within(key: str, age: timedelta) -> bool:
|
||||
"""`key` was marked as a miss less than `age` ago."""
|
||||
try:
|
||||
mtime = (CACHE_DIR / f"{key}.miss").stat().st_mtime
|
||||
# silent-ok: no mark (the usual case).
|
||||
except OSError:
|
||||
return False
|
||||
return time.time() - mtime < age.total_seconds()
|
||||
|
||||
|
||||
def write(key: str, data: bytes, ct: str) -> None:
|
||||
ext = _CT_EXT.get((ct or "").split(";")[0].strip(), "jpg")
|
||||
try:
|
||||
|
||||
+176
-10
@@ -3,13 +3,23 @@ or show (cast_names + the people_text search blob + the plex_people headshot cac
|
||||
orphaned people. Used by BOTH the info-page lazy enrichment (routes.plex) and the background rich
|
||||
re-sync (plex.sync), so the two paths never diverge."""
|
||||
|
||||
from urllib.parse import quote
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import timedelta
|
||||
from typing import Literal
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexPerson
|
||||
from app.plex import imgcache
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Cast/crew photos come from Plex's PUBLIC metadata CDN (no token). We proxy them (host-whitelisted)
|
||||
# so the user's browser makes no third-party request (privacy + self-host CSP friendliness).
|
||||
@@ -19,9 +29,7 @@ PERSON_IMG_HOST = "https://metadata-static.plex.tv/"
|
||||
def person_thumb_url(thumb: str | None) -> str | None:
|
||||
"""Proxied /person-image URL for a Plex CDN headshot, or None when it's missing / off-CDN."""
|
||||
t = str(thumb or "")
|
||||
return (
|
||||
f"/api/plex/person-image?u={quote(t, safe='')}" if t.startswith(PERSON_IMG_HOST) else None
|
||||
)
|
||||
return f"/api/plex/person-image?u={quote(t, safe='')}" if is_portrait_url(t) else None
|
||||
|
||||
|
||||
def cast_from_meta(meta: dict, limit: int = 24) -> list[dict]:
|
||||
@@ -89,14 +97,23 @@ def enrich_row_people(db: Session, row, cast: list[dict]) -> bool:
|
||||
return dirty
|
||||
|
||||
|
||||
# Every credit reference: one person name per (movie/show, credit column). The ONE list of credit
|
||||
# columns, so the orphan prune and the enrichment order never disagree about who is credited.
|
||||
_CREDIT_COLUMNS = [
|
||||
(table, col) for table in ("plex_items", "plex_shows") for col in ("cast_names", "directors")
|
||||
]
|
||||
|
||||
|
||||
def _credit_refs_sql(union: Literal["UNION", "UNION ALL"]) -> str:
|
||||
return f" {union} ".join(
|
||||
f"SELECT jsonb_array_elements_text({col}) FROM {table} WHERE jsonb_typeof({col}) = 'array'"
|
||||
for table, col in _CREDIT_COLUMNS
|
||||
)
|
||||
|
||||
|
||||
# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside
|
||||
# this set are orphans left by a library removal.
|
||||
_REFERENCED_PEOPLE_SQL = """
|
||||
SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array'
|
||||
"""
|
||||
_REFERENCED_PEOPLE_SQL = _credit_refs_sql("UNION")
|
||||
|
||||
|
||||
def prune_orphan_people(db: Session) -> int:
|
||||
@@ -113,3 +130,152 @@ def prune_orphan_people(db: Session) -> int:
|
||||
)
|
||||
db.commit()
|
||||
return res.rowcount or 0
|
||||
|
||||
|
||||
# How often each person is credited across movies, shows and episodes (cast OR director) — the order the
|
||||
# people_enrich job works in, so the faces met most often are local first.
|
||||
_CREDIT_COUNTS_SQL = (
|
||||
f"SELECT ref, count(*) AS n FROM ({_credit_refs_sql('UNION ALL')}) AS refs(ref) GROUP BY ref"
|
||||
)
|
||||
_NEEDS = {
|
||||
# linked to TMDB, bio never fetched
|
||||
"bio": "p.tmdb_person_id IS NOT NULL AND p.tmdb_enriched_at IS NULL",
|
||||
# has a Plex headshot (whether it is cached is the disk's answer, not the row's)
|
||||
"portrait": "p.photo_thumb IS NOT NULL",
|
||||
}
|
||||
|
||||
|
||||
def people_by_credits(
|
||||
db: Session, need: Literal["bio", "portrait"], limit: int | None
|
||||
) -> list[int]:
|
||||
"""plex_people ids that `need` something, the most-credited first (ties by id)."""
|
||||
rows = db.execute(
|
||||
text(
|
||||
f"SELECT p.id FROM plex_people p LEFT JOIN ({_CREDIT_COUNTS_SQL}) c ON c.ref = p.name "
|
||||
f"WHERE {_NEEDS[need]} ORDER BY coalesce(c.n, 0) DESC, p.id"
|
||||
+ (" LIMIT :limit" if limit is not None else "")
|
||||
),
|
||||
{"limit": limit} if limit is not None else {},
|
||||
)
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
# --- Portraits (E30 S6) ---------------------------------------------------------------------------
|
||||
# A Plex CDN headshot is a full-size original (~170 KB on average, measured). Plex's photo transcoder
|
||||
# fits it into this box (~13 KB), which covers the largest place a portrait is drawn (the person page's
|
||||
# 6rem avatar) at 2x density and a raised text size. The box is part of the cache key, like a poster's:
|
||||
# a new box re-caches instead of serving a stale size. Only a resized image is ever written under it —
|
||||
# an original fetched without Plex keeps its own key, and goes once its small twin lands.
|
||||
PORTRAIT_BOX = 320
|
||||
# A headshot Plex answered "not found" for (a dead CDN file) is not asked for again this long, so a
|
||||
# few dead faces never hold up the rest of the prefetch.
|
||||
PORTRAIT_RETRY = timedelta(days=7)
|
||||
# The one shape a Plex CDN headshot has (…/7/people/<hash>.jpg): plain path segments, no query, no
|
||||
# fragment, no dot segment or empty segment (an HTTP client folds `a/../b` and the CDN `a//b` onto one
|
||||
# file). The person-image proxy takes nothing else, so it cannot be made to cache endless spellings of
|
||||
# one file (the image cache has no eviction). Used where a stored headshot is made, served and warmed.
|
||||
_SEGMENT = r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*" # a name, dots only between its parts
|
||||
_PORTRAIT_URL = re.compile(re.escape(PERSON_IMG_HOST) + _SEGMENT + rf"(?:/{_SEGMENT})*")
|
||||
|
||||
|
||||
def is_portrait_url(url: str) -> bool:
|
||||
return _PORTRAIT_URL.fullmatch(url) is not None
|
||||
|
||||
|
||||
def _portrait_keys(url: str) -> tuple[str, str]:
|
||||
"""(the resized portrait's key, the full-size original's key)."""
|
||||
base = "person_" + hashlib.sha1(url.encode()).hexdigest()
|
||||
return f"{base}_{PORTRAIT_BOX}", base
|
||||
|
||||
|
||||
def portrait_cached(url: str) -> tuple[bytes, str] | None:
|
||||
"""The cached portrait: the resized one, else an original kept from a time without Plex."""
|
||||
small, original = _portrait_keys(url)
|
||||
return imgcache.read(small) or imgcache.read(original)
|
||||
|
||||
|
||||
def cache_portrait(plex: PlexClient | None, url: str) -> tuple[bytes, str]:
|
||||
"""Fetch one CDN headshot and cache it. With a Plex client it is resized (the box key; the original
|
||||
is removed); without one the original comes straight from the public CDN and is cached under its own
|
||||
key, so the job still resizes it once Plex is there. Raises PlexError / httpx.HTTPError."""
|
||||
small, original = _portrait_keys(url)
|
||||
if plex is None:
|
||||
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
||||
r = c.get(url)
|
||||
r.raise_for_status()
|
||||
data, ct = r.content, r.headers.get("content-type", "image/jpeg")
|
||||
imgcache.write(original, data, ct)
|
||||
return data, ct
|
||||
data, ct = plex.image_bytes(url, PORTRAIT_BOX, PORTRAIT_BOX)
|
||||
imgcache.write(small, data, ct)
|
||||
imgcache.drop(original)
|
||||
return data, ct
|
||||
|
||||
|
||||
def cdn_url(photo_thumb: str) -> str | None:
|
||||
"""The CDN url inside a stored proxied `/person-image?u=` value (see person_thumb_url)."""
|
||||
_, _, q = photo_thumb.partition("?u=")
|
||||
url = unquote(q)
|
||||
return url if is_portrait_url(url) else None
|
||||
|
||||
|
||||
# Plex failing this many headshots in a row ends a run: it is Plex that is failing, not the files.
|
||||
_PLEX_DOWN_AFTER = 3
|
||||
# The statuses that mean the one file is gone.
|
||||
_DEAD = frozenset({404, 410})
|
||||
|
||||
|
||||
def warm_portraits(db: Session, limit: int) -> dict:
|
||||
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first (E30 S6), so a
|
||||
person's face comes from local disk. Needs Plex (the resize) and the Plex module on; without them
|
||||
nothing is prefetched — full-size originals for every person would cost gigabytes. A headshot Plex
|
||||
answers "not found" for is marked and left for PORTRAIT_RETRY; Plex failing in a row ends the run."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
return {"cached": 0, "skipped": "disabled"}
|
||||
try:
|
||||
plex = PlexClient(db)
|
||||
# silent-ok: no Plex server set up is a setting, not a failure; the run's result says "disabled".
|
||||
except PlexNotConfigured:
|
||||
return {"cached": 0, "skipped": "disabled"}
|
||||
photos: dict[int, str] = {
|
||||
pid: thumb
|
||||
for pid, thumb in db.execute(
|
||||
select(PlexPerson.id, PlexPerson.photo_thumb).where(PlexPerson.photo_thumb.is_not(None))
|
||||
).tuples()
|
||||
if thumb
|
||||
}
|
||||
# `down` counts Plex failing in a row (any answer from Plex breaks it), `plex_failed` in all
|
||||
done = dead = plex_failed = down = 0
|
||||
with plex:
|
||||
for pid in people_by_credits(db, "portrait", None):
|
||||
if done + dead >= limit or down >= _PLEX_DOWN_AFTER:
|
||||
break
|
||||
url = cdn_url(photos.get(pid) or "")
|
||||
if url is None:
|
||||
continue
|
||||
key = _portrait_keys(url)[0]
|
||||
if imgcache.has(key) or imgcache.missed_within(key, PORTRAIT_RETRY):
|
||||
continue
|
||||
try:
|
||||
cache_portrait(plex, url)
|
||||
done, down = done + 1, 0
|
||||
except PlexError as exc:
|
||||
# "Not found" is Plex's answer about this one file (a dead CDN headshot: measured 404,
|
||||
# the CDN itself says 403). Anything else — no answer, a refused token, a 5xx — is
|
||||
# Plex failing, which must neither mark the file nor keep the run going.
|
||||
cause = exc.__cause__
|
||||
if isinstance(cause, httpx.HTTPStatusError) and cause.response.status_code in _DEAD:
|
||||
dead, down = dead + 1, 0 # Plex answered: it is up, the streak is broken
|
||||
imgcache.mark_miss(key)
|
||||
else:
|
||||
plex_failed, down = plex_failed + 1, down + 1
|
||||
log.warning(
|
||||
"a portrait could not be cached",
|
||||
extra=logs.event(
|
||||
"plex.people.portrait_failed",
|
||||
person_id=pid,
|
||||
reason=type(exc.__cause__).__name__,
|
||||
),
|
||||
)
|
||||
failed = dead + plex_failed
|
||||
return {"cached": done, **({"failed": failed} if failed else {})}
|
||||
|
||||
@@ -11,6 +11,7 @@ from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import PlexItem, PlexLibrary
|
||||
from app.plex.genres import hu_genre
|
||||
|
||||
TS_CONFIG = "public.unaccent_simple"
|
||||
|
||||
@@ -183,7 +184,13 @@ def meta_filter_conds(model, p: dict, lang: str = "en") -> list:
|
||||
gsel = split_csv(p.get("genres"))
|
||||
if gsel:
|
||||
gcol = genres_col(model, lang)
|
||||
gc = [gcol.contains([g]) for g in gsel]
|
||||
if lang == "hu":
|
||||
# The HU column holds one Hungarian word per genre (plex/genres.py), so a filter saved while
|
||||
# it offered "Action" is read as "Akció" too. The word as picked still matches as well: a row
|
||||
# the sync has not reached since keeps its English (or Plex's mixed) list until it does.
|
||||
gc = [or_(*(gcol.contains([w]) for w in dict.fromkeys((hu_genre(g), g)))) for g in gsel]
|
||||
else:
|
||||
gc = [gcol.contains([g]) for g in gsel]
|
||||
conds.append(and_(*gc) if p.get("genre_mode") == "all" else or_(*gc))
|
||||
crs = split_csv(p.get("content_ratings"))
|
||||
if crs:
|
||||
|
||||
@@ -32,6 +32,7 @@ window still took the full 145 s and emitted the whole track), so windows are ex
|
||||
zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
||||
"""
|
||||
|
||||
import codecs
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
@@ -217,7 +218,11 @@ _FULL_END_S = 1_000_000_000.0 # stand-in for "to the end of the file" (JSON can
|
||||
|
||||
|
||||
def cache_key(rating_key: str, sub_ord: int) -> str:
|
||||
return f"sub_{rating_key}_{sub_ord}"
|
||||
# Versioned whenever `decode_subtitle` changes what a track decodes to — `sub2_` when legacy code
|
||||
# pages were cached as U+FFFD, `sub3_` when a stray byte, an untagged file or Serbian still went
|
||||
# to the wrong code page. The cache has no eviction, so every track is read again once rather
|
||||
# than a broken one being served for good; the older files are just left behind.
|
||||
return f"sub3_{rating_key}_{sub_ord}"
|
||||
|
||||
|
||||
def _vtt_path(key: str) -> Path:
|
||||
@@ -904,15 +909,90 @@ def _full_pass_start(rating_key: str, targets: dict[int, int]) -> float:
|
||||
return max(0.0, float(f"{min(ends) - WINDOW_PRE_S:.3f}"))
|
||||
|
||||
|
||||
# The legacy code page a subtitle in this language (Plex's ISO 639-2 `languageCode`) was most likely
|
||||
# saved in. Serbian sits here, not with Cyrillic: its releases are overwhelmingly Latin script in
|
||||
# Windows-1250, like Croatian and Bosnian.
|
||||
_CP1250 = {
|
||||
"hun",
|
||||
"ces",
|
||||
"cze",
|
||||
"pol",
|
||||
"slk",
|
||||
"slo",
|
||||
"slv",
|
||||
"hrv",
|
||||
"bos",
|
||||
"srp",
|
||||
"ron",
|
||||
"rum",
|
||||
"sqi",
|
||||
"alb",
|
||||
}
|
||||
_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac"}
|
||||
# Western European languages keep Windows-1252 (cp1250 has no à/è/ñ). An UNTAGGED file falls to
|
||||
# Windows-1250: its ASCII and á/é/í/ó/ö/ú/ü match Windows-1252 byte for byte, and it also gets ő/ű
|
||||
# right, which Windows-1252 turns into õ/û — the library this reads is mostly Hungarian.
|
||||
_CP1252 = {
|
||||
"eng",
|
||||
"fre",
|
||||
"fra",
|
||||
"ger",
|
||||
"deu",
|
||||
"spa",
|
||||
"ita",
|
||||
"por",
|
||||
"dut",
|
||||
"nld",
|
||||
"swe",
|
||||
"dan",
|
||||
"nor",
|
||||
"fin",
|
||||
"ice",
|
||||
"isl",
|
||||
"cat",
|
||||
}
|
||||
# A file that is almost all valid UTF-8 is UTF-8 with a few stray bytes (a pasted Windows-1252 quote),
|
||||
# not a legacy file: accents in a legacy code page are nearly never valid UTF-8 sequences, so a real
|
||||
# legacy file has hardly any. At least this many valid multi-byte characters per invalid byte.
|
||||
_UTF8_VALID_PER_STRAY = 10
|
||||
|
||||
|
||||
def decode_subtitle(raw: bytes, language_code: str | None) -> str:
|
||||
"""A sidecar subtitle file as text. SubRip carries no charset, and a Hungarian release is
|
||||
typically Windows-1250: decoded as UTF-8 with `replace`, every ő/ű/é came out as U+FFFD (found
|
||||
live, 1391 of them in one episode). So: a BOM decides; else valid UTF-8 stays UTF-8 (a legacy
|
||||
code page with accents is almost never valid UTF-8), and so does NEARLY valid UTF-8 — one stray
|
||||
byte must not send a whole UTF-8 file to a legacy code page; else the language's legacy code page."""
|
||||
if raw.startswith(codecs.BOM_UTF8):
|
||||
return raw[len(codecs.BOM_UTF8) :].decode("utf-8", "replace")
|
||||
if raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
|
||||
return raw.decode("utf-16", "replace")
|
||||
try:
|
||||
return raw.decode("utf-8")
|
||||
except UnicodeDecodeError:
|
||||
# silent-ok: not strictly UTF-8 — nearly-UTF-8 or a legacy code page, told apart below.
|
||||
pass
|
||||
lenient = raw.decode("utf-8", "replace")
|
||||
stray = lenient.count("�")
|
||||
valid = sum(1 for ch in lenient if ch > "" and ch != "�")
|
||||
if valid >= stray * _UTF8_VALID_PER_STRAY:
|
||||
return lenient
|
||||
lang = (language_code or "").lower()
|
||||
legacy = "cp1251" if lang in _CP1251 else "cp1252" if lang in _CP1252 else "cp1250"
|
||||
return raw.decode(legacy, "replace")
|
||||
|
||||
|
||||
def _sidecar_vtt(db: Session, stream: dict) -> str:
|
||||
"""An external subtitle file, fetched through Plex and converted to WebVTT."""
|
||||
"""An external subtitle file, fetched through Plex and converted to WebVTT. Its bytes are made
|
||||
UTF-8 first (`decode_subtitle`), on the ffmpeg path too: ffmpeg reads a text subtitle as UTF-8."""
|
||||
with PlexClient(db) as plex:
|
||||
raw = plex.raw_get(stream["key"])
|
||||
text = decode_subtitle(raw, stream.get("languageCode"))
|
||||
codec = (stream.get("codec") or stream.get("format") or "").lower()
|
||||
if codec in ("srt", "subrip", ""):
|
||||
return srt_to_vtt(raw.decode("utf-8", "replace"))
|
||||
return srt_to_vtt(text)
|
||||
with tempfile.NamedTemporaryFile(suffix="." + (codec or "sub"), delete=False) as fh:
|
||||
fh.write(raw)
|
||||
fh.write(text.encode("utf-8"))
|
||||
tmp = fh.name
|
||||
try:
|
||||
return ffmpeg_to_vtt(tmp, None)
|
||||
|
||||
+64
-19
@@ -15,6 +15,7 @@ dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are
|
||||
item-detail/player path (P2), not here.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
@@ -24,6 +25,7 @@ from app import logs, sysconfig
|
||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||
from app.plex import detail, people
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.genres import hu_genres
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
@@ -448,6 +450,10 @@ def _apply_facet_fields(row, meta: dict) -> None:
|
||||
row.studio = (meta.get("studio") or None) and str(meta.get("studio"))[:255]
|
||||
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
|
||||
row.genres = _tags(meta, "Genre")
|
||||
# Every row has Hungarian genres, not only one with an HU overlay twin (the overlay pass, which runs
|
||||
# after this, puts the twin's own list over it): the HU genre facets and filter read this column,
|
||||
# and an English word there could never match a filter read in Hungarian.
|
||||
row.genres_hu = hu_genres(row.genres)
|
||||
row.directors = _tags(meta, "Director")
|
||||
# The listing carries only the top ~20 cast, so re-applying it would DOWNGRADE an already-enriched
|
||||
# full cast (up to 24, grown by the rich backfill/info-page enrich), silently breaking the
|
||||
@@ -659,35 +665,72 @@ def _sync_overlay(
|
||||
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
||||
) -> None:
|
||||
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/genres differ
|
||||
by language; the language-neutral people_text feeds both search vectors.
|
||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/tagline/genres
|
||||
differ by language; the language-neutral people_text feeds both search vectors. A show section's
|
||||
episodes are overlaid too (E30): the HU section lists every episode with its Hungarian title and
|
||||
summary, and the episode rows live in the EN show library.
|
||||
|
||||
This is a full RECONCILE: every guid-bearing EN row's HU fields are cleared first, then the HU
|
||||
section repopulates the matches — so an EN row whose HU twin vanished (or was re-guid'd) falls back
|
||||
to EN instead of keeping stale Hungarian metadata. HU items with no EN match are skipped."""
|
||||
if lib.kind == "movie":
|
||||
rows = db.execute(
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.guid.is_not(None),
|
||||
passes = [
|
||||
(
|
||||
_TYPE_MOVIE,
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.guid.is_not(None),
|
||||
),
|
||||
)
|
||||
).scalars()
|
||||
item_type = _TYPE_MOVIE
|
||||
]
|
||||
else:
|
||||
rows = db.execute(
|
||||
select(PlexShow).where(PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None))
|
||||
).scalars()
|
||||
item_type = _TYPE_SHOW
|
||||
passes = [
|
||||
(
|
||||
_TYPE_SHOW,
|
||||
select(PlexShow).where(
|
||||
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
|
||||
),
|
||||
),
|
||||
(
|
||||
_TYPE_EPISODE,
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "episode",
|
||||
PlexItem.guid.is_not(None),
|
||||
),
|
||||
),
|
||||
]
|
||||
for item_type, rows_q in passes:
|
||||
_overlay_pass(db, plex, lib, item_type, db.execute(rows_q).scalars(), stats)
|
||||
db.flush()
|
||||
|
||||
|
||||
def _overlay_pass(
|
||||
db: Session,
|
||||
plex: PlexClient,
|
||||
lib: PlexLibrary,
|
||||
item_type: int,
|
||||
rows: Iterable[PlexItem] | Iterable[PlexShow],
|
||||
stats: dict,
|
||||
) -> None:
|
||||
"""One kind of the overlay (movies, shows or episodes) — see `_sync_overlay`."""
|
||||
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
||||
by_guid: dict[str, list] = {}
|
||||
by_guid: dict[str, list[PlexItem | PlexShow]] = {}
|
||||
for r in rows:
|
||||
by_guid.setdefault(r.guid, []).append(r)
|
||||
if r.guid: # the passes select guid-bearing rows only
|
||||
by_guid.setdefault(r.guid, []).append(r)
|
||||
# Reconcile step: clear HU fields on all guid-bearing rows. Only rows that actually held HU data
|
||||
# become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is cheap; matches are re-set below.
|
||||
# The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking one, from
|
||||
# its own English list — otherwise the HU genre list mixed "Drama" (no twin) with "Dráma".
|
||||
episodes = item_type == _TYPE_EPISODE
|
||||
for group in by_guid.values():
|
||||
for row in group:
|
||||
row.title_hu = row.summary_hu = row.genres_hu = None
|
||||
row.title_hu = row.summary_hu = None
|
||||
if not episodes:
|
||||
row.tagline_hu = None
|
||||
row.genres_hu = hu_genres(row.genres)
|
||||
seen: set[str] = (
|
||||
set()
|
||||
) # first HU entry for a guid wins; a dup HU item can't overwrite or double-count
|
||||
@@ -699,13 +742,15 @@ def _sync_overlay(
|
||||
seen.add(g)
|
||||
title_hu = (meta.get("title") or "").strip() or None
|
||||
summary_hu = meta.get("summary")
|
||||
genres_hu = _tags(meta, "Genre")
|
||||
tagline_hu = (meta.get("tagline") or "").strip() or None
|
||||
genres_hu = hu_genres(_tags(meta, "Genre"))
|
||||
for row in targets:
|
||||
row.title_hu = title_hu
|
||||
row.summary_hu = summary_hu
|
||||
row.genres_hu = genres_hu
|
||||
if not episodes:
|
||||
row.tagline_hu = tagline_hu
|
||||
row.genres_hu = genres_hu or row.genres_hu
|
||||
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
||||
db.flush()
|
||||
|
||||
|
||||
def _upsert_collection(
|
||||
|
||||
+180
-27
@@ -10,8 +10,10 @@ One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `
|
||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||
"""
|
||||
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import cast
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import func, select
|
||||
@@ -21,6 +23,8 @@ from sqlalchemy.orm import Session
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import PlexItem, PlexPerson, PlexShow
|
||||
from app.plex.cards import display_title
|
||||
from app.plex.people import people_by_credits, warm_portraits
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||
@@ -124,9 +128,17 @@ class TmdbClient:
|
||||
"""Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find."""
|
||||
return self._get(f"/find/{external_id}", {"external_source": source})
|
||||
|
||||
def countries(self) -> list[dict] | None:
|
||||
"""TMDB's country list: `[{iso_3166_1, english_name, native_name}]` (a JSON array, which
|
||||
`_get`'s dict annotation does not describe)."""
|
||||
return cast("list[dict] | None", self._get("/configuration/countries"))
|
||||
|
||||
def fetch_person(self, person_id: int) -> dict | None:
|
||||
"""One person's bio + combined credits (E5 S6), in a single append_to_response request."""
|
||||
return self._get(f"/person/{person_id}", {"append_to_response": "combined_credits"})
|
||||
"""One person's bio + combined credits (E5 S6) + the bio's translations (E30 S4), in a single
|
||||
append_to_response request."""
|
||||
return self._get(
|
||||
f"/person/{person_id}", {"append_to_response": "combined_credits,translations"}
|
||||
)
|
||||
|
||||
|
||||
def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None:
|
||||
@@ -254,8 +266,12 @@ def parse_tmdb(payload: dict, media_type: str) -> dict:
|
||||
c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")
|
||||
][:8]
|
||||
or None,
|
||||
# The ISO code beside TMDB's English name (E30): the client names the country in the reader's
|
||||
# language from the code; the name is the fallback for a code it cannot name.
|
||||
"production_countries": [
|
||||
c.get("name") for c in (payload.get("production_countries") or []) if c.get("name")
|
||||
{"code": c.get("iso_3166_1") or None, "name": c["name"]}
|
||||
for c in (payload.get("production_countries") or [])
|
||||
if c.get("name")
|
||||
]
|
||||
or None,
|
||||
"recommendations": recs or None,
|
||||
@@ -301,8 +317,8 @@ def _link_people(db: Session, links: dict[str, int]) -> None:
|
||||
|
||||
|
||||
def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the remainder —
|
||||
mirrors _enrich_people_batch). A transient TmdbError leaves the watermark NULL for retry; a
|
||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the
|
||||
remainder). A transient TmdbError leaves the watermark NULL for retry; a
|
||||
definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched
|
||||
every run. One commit at the end. Returns the number processed (matched OR definitively unmatched)."""
|
||||
movies = (
|
||||
@@ -356,13 +372,62 @@ def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||
return processed
|
||||
|
||||
|
||||
def _country(c: dict | str) -> dict:
|
||||
"""One stored production country as served: `{code, name}`. A row enriched before E30 stored the
|
||||
bare English name; until `backfill_country_codes` reaches it, it serves with no code (the client
|
||||
shows the name)."""
|
||||
return c if isinstance(c, dict) else {"code": None, "name": c}
|
||||
|
||||
|
||||
def backfill_country_codes(db: Session, tmdb: TmdbClient) -> int:
|
||||
"""Give the countries stored before E30 (bare English names) their ISO codes, from TMDB's own
|
||||
country list — the names came from that list, so they match it exactly. One request, and only
|
||||
while such a row is left: re-fetching each title instead would take the enrichment batch days.
|
||||
A name the list does not hold keeps no code. Returns the number of rows rewritten."""
|
||||
stale = [
|
||||
row
|
||||
for model in (PlexItem, PlexShow)
|
||||
for row in db.execute(
|
||||
select(model).where(func.jsonb_typeof(model.production_countries[0]) == "string")
|
||||
).scalars()
|
||||
]
|
||||
if not stale:
|
||||
return 0
|
||||
codes = {
|
||||
c["english_name"]: c["iso_3166_1"]
|
||||
for c in tmdb.countries() or []
|
||||
if c.get("english_name") and c.get("iso_3166_1")
|
||||
}
|
||||
for row in stale:
|
||||
row.production_countries = [
|
||||
{"code": codes.get(c), "name": c} if isinstance(c, str) else c
|
||||
for c in row.production_countries or []
|
||||
]
|
||||
db.commit()
|
||||
return len(stale)
|
||||
|
||||
|
||||
def run_enrich(db: Session) -> dict:
|
||||
"""Scheduler entry point. No-op (never an error) when the TMDB key isn't configured."""
|
||||
if not sysconfig.get_str(db, "tmdb_api_key"):
|
||||
return {"enriched": 0, "skipped": "tmdb_api_key not set"}
|
||||
with TmdbClient(db) as tmdb:
|
||||
coded = backfill_country_codes(db, tmdb)
|
||||
n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch)
|
||||
return {"enriched": n}
|
||||
return {"enriched": n, **({"countries_coded": coded} if coded else {})}
|
||||
|
||||
|
||||
def run_people_enrich(db: Session) -> dict:
|
||||
"""Scheduler entry point of people_enrich (E30 S6): a batch of person bios (the most-credited
|
||||
first) and a batch of portraits. Each half is a no-op — never an error — without its TMDB key /
|
||||
without Plex."""
|
||||
batch = settings.people_enrich_batch
|
||||
if sysconfig.get_str(db, "tmdb_api_key"):
|
||||
with TmdbClient(db) as tmdb:
|
||||
bios = enrich_people_batch(db, tmdb, batch)
|
||||
else:
|
||||
bios = {"skipped": "tmdb_api_key not set"}
|
||||
return {"bios": bios, "portraits": warm_portraits(db, batch)}
|
||||
|
||||
|
||||
def resolve_library_guids(
|
||||
@@ -406,10 +471,11 @@ def resolve_library_guids(
|
||||
return movies, shows
|
||||
|
||||
|
||||
def resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]:
|
||||
def resolve_recs(db: Session, recs: list[dict] | None, *, lang: str) -> list[dict]:
|
||||
"""Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:<id>`, bucketed
|
||||
by media_type — see resolve_library_guids) and, if so, its rating_key — so the client can deep-link
|
||||
the ones we have and merely display the rest."""
|
||||
the ones we have and merely display the rest. One we have is named as the library names it, in
|
||||
`lang` (E30); one we lack keeps the title TMDB gave."""
|
||||
if not recs:
|
||||
return []
|
||||
movies, shows = resolve_library_guids(db, recs)
|
||||
@@ -417,13 +483,19 @@ def resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]:
|
||||
for r in recs:
|
||||
g = f"tmdb:{r.get('tmdb_id')}"
|
||||
row = shows.get(g) if r.get("media_type") == "tv" else movies.get(g)
|
||||
named = {"title": display_title(row, lang)} if row is not None else {}
|
||||
out.append(
|
||||
{**r, "in_library": row is not None, "rating_key": row.rating_key if row else None}
|
||||
{
|
||||
**r,
|
||||
**named,
|
||||
"in_library": row is not None,
|
||||
"rating_key": row.rating_key if row else None,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def serve_block(db: Session, row) -> dict | None:
|
||||
def serve_block(db: Session, row, *, lang: str) -> dict | None:
|
||||
"""The TMDB extras for a movie/show detail response, served from the row (no live TMDB call).
|
||||
None until the row has been TMDB-enriched (the client then shows nothing extra)."""
|
||||
if row.tmdb_enriched_at is None:
|
||||
@@ -437,30 +509,58 @@ def serve_block(db: Session, row) -> dict | None:
|
||||
"keywords": row.keywords or [],
|
||||
"crew": row.crew or [],
|
||||
"studios": row.studios or [],
|
||||
"production_countries": row.production_countries or [],
|
||||
"recommendations": resolve_recs(db, row.recommendations),
|
||||
"production_countries": [_country(c) for c in row.production_countries or []],
|
||||
"recommendations": resolve_recs(db, row.recommendations, lang=lang),
|
||||
"backdrops": row.backdrops or [],
|
||||
}
|
||||
|
||||
|
||||
# --- Person enrichment (E5 S6) -------------------------------------------------------------------
|
||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once (the first person-page open)
|
||||
# and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once — by the people_enrich job, or
|
||||
# by the first person-page open if the job has not reached them yet — and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||
# endpoint resolves them to library rows by guid at serve time (like recommendations).
|
||||
|
||||
_KNOWN_FOR_MAX = 15
|
||||
# The candidate pool kept per person. Only the titles that are in the library are shown (at most
|
||||
# routes.plex.people._KNOWN_FOR_SHOWN), so the pool is deeper than the rail: a star's 15 most-voted
|
||||
# credits are often not the 15 a given library holds.
|
||||
_KNOWN_FOR_MAX = 40
|
||||
# TMDB genre ids of the shows a person visits as a guest (News, Reality, Talk). Ranked by popularity,
|
||||
# talk-show visits crowded out the films (Tom Cruise: 14 of 15 were talk shows; E30 S6).
|
||||
_GUEST_GENRES = {10763, 10764, 10767}
|
||||
# A credit in which the person plays themselves ("Self", "Himself - Guest", "Herself (archive footage)").
|
||||
_SELF_CREDIT = re.compile(r"^\s*(self|himself|herself|themselves)\b", re.IGNORECASE)
|
||||
# The crew jobs that make a title the person's own work (a director is known for what they directed).
|
||||
_KNOWN_FOR_CREW_JOBS = {"Director", "Creator"}
|
||||
|
||||
|
||||
def _known_for_credit(c: dict, crew: bool) -> bool:
|
||||
if crew:
|
||||
return c.get("job") in _KNOWN_FOR_CREW_JOBS
|
||||
if _SELF_CREDIT.match(c.get("character") or ""):
|
||||
return False
|
||||
return not _GUEST_GENRES.intersection(c.get("genre_ids") or [])
|
||||
|
||||
|
||||
def parse_person(payload: dict) -> dict:
|
||||
"""TMDB /person/{id} (+combined_credits) → the persisted bio fields. `known_for` = the person's
|
||||
most-popular movie/tv credits (deduped by id), top `_KNOWN_FOR_MAX`, resolved to library later."""
|
||||
credits = (payload.get("combined_credits") or {}).get("cast") or []
|
||||
most-voted movie/tv credits — roles and directed works, not guest visits as themselves — deduped,
|
||||
the top `_KNOWN_FOR_MAX`, resolved to library rows at serve time. Ranked by vote count, as TMDB's
|
||||
own "Known for" is: popularity is this week's attention, which a talk-show visit wins."""
|
||||
combined = payload.get("combined_credits") or {}
|
||||
credits = [c for c in combined.get("cast") or [] if _known_for_credit(c, crew=False)] + [
|
||||
c for c in combined.get("crew") or [] if _known_for_credit(c, crew=True)
|
||||
]
|
||||
# Dedup on (id, media_type), not the bare id: TMDB movie ids and tv ids are independent sequences
|
||||
# (id 603 can be both a movie and a show — see resolve_library_guids), so a same-id movie+show pair
|
||||
# is two distinct credits and both must survive.
|
||||
seen: set[tuple[int, str]] = set()
|
||||
known: list[dict] = []
|
||||
for c in sorted(credits, key=lambda c: c.get("popularity") or 0, reverse=True):
|
||||
ranked = sorted(
|
||||
credits,
|
||||
key=lambda c: (c.get("vote_count") or 0, c.get("popularity") or 0),
|
||||
reverse=True,
|
||||
)
|
||||
for c in ranked:
|
||||
tid, mt = c.get("id"), c.get("media_type")
|
||||
if not tid or mt not in ("movie", "tv") or (tid, mt) in seen:
|
||||
continue
|
||||
@@ -470,6 +570,7 @@ def parse_person(payload: dict) -> dict:
|
||||
break
|
||||
return {
|
||||
"biography": (payload.get("biography") or "").strip() or None,
|
||||
"biography_hu": _translated_bio(payload, "hu"),
|
||||
"birthday": payload.get("birthday") or None,
|
||||
"deathday": payload.get("deathday") or None,
|
||||
"place_of_birth": payload.get("place_of_birth") or None,
|
||||
@@ -479,8 +580,17 @@ def parse_person(payload: dict) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _translated_bio(payload: dict, lang: str) -> str | None:
|
||||
"""The `lang` biography from the appended `translations` (TMDB leaves an untranslated one blank)."""
|
||||
for tr in (payload.get("translations") or {}).get("translations") or []:
|
||||
if tr.get("iso_639_1") == lang:
|
||||
return ((tr.get("data") or {}).get("biography") or "").strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def apply_person(row: PlexPerson, parsed: dict) -> None:
|
||||
row.biography = parsed["biography"]
|
||||
row.biography_hu = parsed["biography_hu"]
|
||||
row.birthday = parsed["birthday"]
|
||||
row.deathday = parsed["deathday"]
|
||||
row.place_of_birth = parsed["place_of_birth"]
|
||||
@@ -490,11 +600,22 @@ def apply_person(row: PlexPerson, parsed: dict) -> None:
|
||||
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _fetch_person_into(tmdb: TmdbClient, person: PlexPerson) -> None:
|
||||
"""Fetch one person from TMDB and write the row (not committed). A 404 (deleted person) stamps the
|
||||
watermark with empty fields so it isn't fetched again. Raises TmdbError on a transient failure,
|
||||
leaving the row untouched and `tmdb_enriched_at` NULL for a retry."""
|
||||
payload = tmdb.fetch_person(cast(int, person.tmdb_person_id))
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
else:
|
||||
apply_person(person, parse_person(payload))
|
||||
|
||||
|
||||
def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
"""Lazily fetch + persist one person's TMDB bio (first person-page open), committing the write so it
|
||||
survives regardless of the read path. No-op (False) without a key or a `tmdb_person_id`; a transient
|
||||
error leaves `tmdb_enriched_at` NULL so the next open retries. A 404 (deleted person) stamps the
|
||||
watermark with empty fields so we don't refetch it every open."""
|
||||
"""Lazily fetch + persist one person's TMDB bio (a person page opened before the background job
|
||||
reached them), committing the write so it survives regardless of the read path. No-op (False)
|
||||
without a key or a `tmdb_person_id`; a transient error leaves `tmdb_enriched_at` NULL so the next
|
||||
open retries."""
|
||||
if not person.tmdb_person_id or not sysconfig.get_str(db, "tmdb_api_key"):
|
||||
return False
|
||||
try:
|
||||
@@ -502,7 +623,7 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
# longer blocks a serve; still cap the outbound TMDB call with a short timeout so a hanging TMDB
|
||||
# doesn't tie up a threadpool worker for the default 30s.
|
||||
with TmdbClient(db, timeout=8.0) as tmdb:
|
||||
payload = tmdb.fetch_person(person.tmdb_person_id)
|
||||
_fetch_person_into(tmdb, person)
|
||||
except TmdbError as exc:
|
||||
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
||||
# front of the user right now shows an empty biography with nothing to say why.
|
||||
@@ -515,9 +636,41 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
),
|
||||
)
|
||||
return False
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
else:
|
||||
apply_person(person, parse_person(payload))
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
# Lookups failing this many people in a row end a people batch: TMDB itself is degraded, and hammering
|
||||
# it helps nobody. One person failing alone is skipped, so an id TMDB keeps refusing never stalls the rest.
|
||||
_PEOPLE_FAILS_IN_A_ROW = 3
|
||||
|
||||
|
||||
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
||||
"""Fetch up to `limit` linked, not-yet-enriched people (E30 S6), the most-credited first, so a
|
||||
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
||||
lands: a page open during the run already sees it. A failed lookup is skipped (left unstamped for the
|
||||
next run); _PEOPLE_FAILS_IN_A_ROW of them in a row end the batch."""
|
||||
ids = people_by_credits(db, "bio", limit)
|
||||
done = failed = in_a_row = 0
|
||||
for pid in ids:
|
||||
if in_a_row >= _PEOPLE_FAILS_IN_A_ROW:
|
||||
break
|
||||
person = db.get(PlexPerson, pid)
|
||||
# A page open may have enriched this person since the select: skip, don't fetch twice.
|
||||
if person is None or person.tmdb_enriched_at is not None:
|
||||
continue
|
||||
try:
|
||||
_fetch_person_into(tmdb, person)
|
||||
except TmdbError as exc:
|
||||
db.rollback()
|
||||
failed, in_a_row = failed + 1, in_a_row + 1
|
||||
log.warning(
|
||||
"a TMDB person lookup in the batch failed",
|
||||
extra=logs.event(
|
||||
"plex.tmdb.people_batch_failed", person_id=pid, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
continue
|
||||
db.commit()
|
||||
done, in_a_row = done + 1, 0
|
||||
return {"enriched": done, **({"failed": failed} if failed else {})}
|
||||
|
||||
@@ -6,7 +6,7 @@ and "fetch a row this user owns or 404" idioms have one definition, one status c
|
||||
one existence-leak policy.
|
||||
"""
|
||||
|
||||
from typing import Any, TypeVar
|
||||
from typing import IO, Any, TypeVar
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -28,10 +28,13 @@ def not_found(what: str = "generic") -> HTTPException:
|
||||
return HTTPException(status_code=404, detail={"code": "not_found", "reason": what})
|
||||
|
||||
|
||||
def get_or_404(db: Session, model: type[M], id_: Any, *, what: str = "generic") -> M:
|
||||
def get_or_404(
|
||||
db: Session, model: type[M], id_: Any, *, what: str = "generic", lock: bool = False
|
||||
) -> M:
|
||||
"""Fetch a row by primary key or raise 404. Replaces the `db.get(Model, id); if None: 404`
|
||||
idiom repeated across routes."""
|
||||
obj = db.get(model, id_)
|
||||
idiom repeated across routes. `lock` takes it FOR UPDATE (bypassing the identity map), so two
|
||||
requests that read-then-write its children (a playlist's item positions) run one after the other."""
|
||||
obj = db.get(model, id_, with_for_update=True if lock else None)
|
||||
if obj is None:
|
||||
raise not_found(what)
|
||||
return obj
|
||||
@@ -45,12 +48,13 @@ def owned_or_404(
|
||||
*,
|
||||
owner_attr: str = "user_id",
|
||||
what: str = "generic",
|
||||
lock: bool = False,
|
||||
) -> M:
|
||||
"""Fetch a row by primary key that `user` owns, or raise 404. A row owned by someone
|
||||
"""Fetch a row by primary key that `user` owns, or raise 404 (`lock`: see `get_or_404`). A row owned by someone
|
||||
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
||||
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
|
||||
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
|
||||
obj = get_or_404(db, model, id_, what=what)
|
||||
obj = get_or_404(db, model, id_, what=what, lock=lock)
|
||||
if getattr(obj, owner_attr) != user.id:
|
||||
raise not_found(what)
|
||||
return obj
|
||||
@@ -79,6 +83,47 @@ async def capped_body(request: Request, limit: int) -> bytes:
|
||||
return bytes(body)
|
||||
|
||||
|
||||
async def capped_stream_to(
|
||||
request: Request, out: IO[bytes], limit: int, refusal: HTTPException
|
||||
) -> int:
|
||||
"""`capped_body`'s sibling for a body too large to hold: the same counting rule, written
|
||||
straight to `out` instead of accumulated, returning the size.
|
||||
|
||||
Two deliberate differences from `capped_body`. It never builds a `bytearray` — the upload route
|
||||
admits 500 MiB by default, so holding the body would trade a disk problem for a memory one. And
|
||||
the refusal is the CALLER'S, passed in rather than assumed: `capped_body`'s flat 413 is right
|
||||
for a screenshot, while the upload route answers 422 `upload_too_large` carrying the admin
|
||||
knob's own number, which is what its dialog shows the user. One counting rule, two answers,
|
||||
neither of them a second copy of the loop.
|
||||
|
||||
The caller owns `out` and owns cleaning it up: a body refused half-way has already written what
|
||||
arrived before the limit was passed.
|
||||
|
||||
**The refusal DRAINS the rest of the body before it is raised, and that is not politeness.**
|
||||
Answering mid-upload and closing the connection is what the client sees as a network failure,
|
||||
not as a refusal: the browser's XHR fires `onerror` before it can read the response, so a file
|
||||
over the limit reported itself as "the server couldn't be reached" instead of naming the limit —
|
||||
measured in a real browser, not reasoned about. Reading the remainder to nowhere lets the client
|
||||
finish sending, read the 422 and say the true thing.
|
||||
|
||||
The bytes are discarded, never written, and the whole request is already bounded by
|
||||
`bodylimit`'s per-route ceiling — so the worst case is bandwidth an authenticated caller was
|
||||
sending anyway, against the old behaviour which wrote exactly those bytes to disk."""
|
||||
size = 0
|
||||
over = False
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if over:
|
||||
continue # draining — see the docstring; counted, not stored
|
||||
if size > limit:
|
||||
over = True
|
||||
continue
|
||||
out.write(chunk)
|
||||
if over:
|
||||
raise refusal
|
||||
return size
|
||||
|
||||
|
||||
def issue_refused(exc: gitea.GiteaError) -> HTTPException:
|
||||
"""The refusal an issue-tracker failure (E16) answers as — the send, the preview and the
|
||||
admin's connection test alike. Literal reasons, one per case: the frontend's contract test reads
|
||||
|
||||
@@ -834,7 +834,11 @@ def reset_demo(admin: User = Depends(admin_user), db: Session = Depends(get_db))
|
||||
demo = get_or_create_demo_user(db)
|
||||
seeded = reset_demo_state(db, demo)
|
||||
audit.record(
|
||||
db, admin.id, AuditAction.DEMO_RESET, summary=f"{seeded} sample playlists re-seeded"
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.DEMO_RESET,
|
||||
summary=f"{seeded} sample playlists re-seeded",
|
||||
after={"playlists_seeded": seeded}, # the count as data: named in the reader's language
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
||||
|
||||
@@ -66,6 +66,12 @@ def list_audit(
|
||||
def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Clear the audit log. Leaves ONE row recording who cleared it + how many (tamper-evidence)."""
|
||||
n = db.execute(delete(AuditLog)).rowcount or 0
|
||||
audit.record(db, admin.id, AuditAction.AUDIT_CLEAR, summary=f"Cleared {n} audit entries")
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.AUDIT_CLEAR,
|
||||
summary=f"Cleared {n} audit entries",
|
||||
after={"cleared": n}, # the count as data: the log names it in the reader's language
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "cleared": n}
|
||||
|
||||
@@ -39,7 +39,7 @@ from app.auth import (
|
||||
)
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import AuthToken, DemoWhitelist, Invite, OAuthToken, User
|
||||
from app.models import AuthToken, DemoWhitelist, Invite, OAuthToken, User, display_text
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import encrypt, hash_password, hash_token, verify_password
|
||||
from app.utils import canonical_email, valid_email
|
||||
@@ -472,7 +472,7 @@ async def callback(
|
||||
if email:
|
||||
user.google_email = email
|
||||
_sync_google_email(db, user, email)
|
||||
user.display_name = userinfo.get("name")
|
||||
user.display_name = display_text(userinfo.get("name"))
|
||||
user.avatar_url = userinfo.get("picture")
|
||||
# ADMIN_EMAILS (env) is the bootstrap admin list and always wins, so the configured admin can't
|
||||
# be locked out. Everyone else's role is managed in the admin UI and stored in the DB, so a
|
||||
@@ -594,7 +594,7 @@ def _complete_link(
|
||||
if linked_email:
|
||||
user.google_email = linked_email # see the sign-in callback: the other half of the identity
|
||||
if not user.display_name:
|
||||
user.display_name = userinfo.get("name")
|
||||
user.display_name = display_text(userinfo.get("name"))
|
||||
if not user.avatar_url:
|
||||
user.avatar_url = userinfo.get("picture")
|
||||
_store_token(db, user, token)
|
||||
|
||||
@@ -9,6 +9,8 @@ from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.collation import collated, sorted_words
|
||||
from app.mailtext import lang_for_user
|
||||
from app import logs, quota
|
||||
from app.auth import admin_user, current_user, require_human, require_write_scope
|
||||
from app.db import get_db
|
||||
@@ -28,7 +30,7 @@ from app.models import (
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.sync.explore import explore_ingest_page
|
||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.sync.subscriptions import apply_channel_details, details_stale
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
@@ -44,7 +46,7 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_
|
||||
select(Subscription, Channel)
|
||||
.join(Channel, Channel.id == Subscription.channel_id)
|
||||
.where(Subscription.user_id == user.id)
|
||||
.order_by(Subscription.priority.desc(), func.lower(Channel.title))
|
||||
.order_by(Subscription.priority.desc(), collated(Channel.title, lang_for_user(user)))
|
||||
).all()
|
||||
|
||||
# Per-channel aggregates over the shared catalog in ONE grouped pass: stored count, last
|
||||
@@ -139,7 +141,7 @@ def _discovery_rows(db: Session, user: User) -> list:
|
||||
return db.execute(
|
||||
q.group_by(Channel.id).order_by(
|
||||
func.count(func.distinct(PlaylistItem.video_id)).desc(),
|
||||
func.lower(Channel.title),
|
||||
collated(Channel.title, lang_for_user(user)),
|
||||
)
|
||||
).all()
|
||||
|
||||
@@ -190,12 +192,16 @@ def discover_channels(
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
apply_channel_details(db, yt.get_channels(need))
|
||||
apply_channel_details(db, yt.get_channels(need), need)
|
||||
db.commit()
|
||||
# Match _discovery_rows' ORDER BY exactly: video-count DESC, then title (NULLs LAST,
|
||||
# as Postgres orders them — coercing NULL→"" would wrongly float untitled channels up).
|
||||
# Match _discovery_rows' ORDER BY exactly: video-count DESC, then title in the reader's
|
||||
# alphabet (NULLs LAST, as Postgres orders them — coercing NULL→"" would wrongly float
|
||||
# untitled channels up).
|
||||
titles = [r[0].title for r in rows if r[0].title is not None]
|
||||
alpha = {w: i for i, w in enumerate(sorted_words(db, titles, lang_for_user(user)))}
|
||||
rows = sorted(
|
||||
rows, key=lambda r: (-int(r[1]), r[0].title is None, (r[0].title or "").lower())
|
||||
rows,
|
||||
key=lambda r: (-int(r[1]), r[0].title is None, alpha.get(r[0].title or "", 0)),
|
||||
)
|
||||
except YouTubeError as exc:
|
||||
log.warning(
|
||||
@@ -296,17 +302,20 @@ def channel_detail(
|
||||
"""One channel's "About" detail for the channel page header/About tab, plus this user's
|
||||
relationship to it (subscribed / explored) and how many of its videos are in the catalog.
|
||||
Lazily fills the About metadata (view count, join date, banner, links) the first time it's
|
||||
needed — a stub, or a channel last enriched before those fields existed (published_at is a
|
||||
reliable sentinel: every real channel has a creation date). The enrich uses the API key
|
||||
(1 unit, cached after), so it's skipped for the quota-less demo account."""
|
||||
needed (a stub) and refreshes it once it is stale (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the
|
||||
subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit,
|
||||
cached after), so it's skipped for the quota-less demo account."""
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if channel.published_at is None and not user.is_demo:
|
||||
# No `published_at is None` sentinel: a terminated channel never gets one, so it re-fetched on
|
||||
# every view; rows enriched before that field existed are long past DETAILS_MAX_AGE anyway.
|
||||
stale = details_stale(channel, datetime.now(timezone.utc))
|
||||
if stale and not user.is_demo:
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||
db.commit()
|
||||
channel = db.get(Channel, channel_id)
|
||||
except YouTubeError as exc:
|
||||
@@ -412,12 +421,10 @@ def explore_channel(
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
# Enrich for the About panel + the uploads playlist id (needed to ingest videos).
|
||||
if (
|
||||
channel.details_synced_at is None
|
||||
or channel.uploads_playlist_id is None
|
||||
or channel.published_at is None
|
||||
):
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
# Stale, not "a field is missing": a terminated channel never gets an uploads playlist
|
||||
# or a creation date, and asking again on every explore would spend quota on a miss.
|
||||
if details_stale(channel, datetime.now(timezone.utc)):
|
||||
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||
db.commit()
|
||||
channel = db.get(Channel, channel_id)
|
||||
result = explore_ingest_page(db, yt, channel, page_token)
|
||||
@@ -590,7 +597,7 @@ def subscribe(
|
||||
# must not block recording the subscription the user asked for.
|
||||
if channel.details_synced_at is None:
|
||||
try:
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||
except YouTubeError as exc:
|
||||
log.warning(
|
||||
"subscribe: stub enrich failed: %s",
|
||||
@@ -658,7 +665,7 @@ def list_blocked(user: User = Depends(current_user), db: Session = Depends(get_d
|
||||
select(Channel)
|
||||
.join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
|
||||
.where(BlockedChannel.user_id == user.id)
|
||||
.order_by(func.lower(Channel.title))
|
||||
.order_by(collated(Channel.title, lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
|
||||
@@ -22,7 +22,7 @@ from sqlalchemy.orm import Session
|
||||
from app import audit, debugsession, logs, logsink
|
||||
from app.audit import AuditAction
|
||||
from app.auth import require_human
|
||||
from app.db import get_db
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.issues import bundle, gitea
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.refusal import refused
|
||||
@@ -335,7 +335,13 @@ def _upload_target(
|
||||
return owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
|
||||
|
||||
async def _image_body(request: Request) -> bytes:
|
||||
async def _image_body(request: Request, db: Session = Depends(get_db)) -> bytes:
|
||||
# The owner check above has already run, and it was a SELECT — so this session is holding a
|
||||
# pooled connection `idle in transaction` and would hold it for the whole 10 MiB upload, from a
|
||||
# client that may be on a phone. Hand it back first; the handler's own work re-takes one. The
|
||||
# `session` object the handler holds is expired by this and reloads on its next attribute read,
|
||||
# which is the correct moment to touch the database again.
|
||||
release_read_connection(db)
|
||||
return await capped_body(request, debugsession.SCREENSHOT_MAX_BYTES)
|
||||
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, diagnostics, logs, logsink
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.models import DebugSession, LogEvent, User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import capped_body
|
||||
@@ -73,7 +73,7 @@ class ClientEventsIn(BaseModel):
|
||||
events: list[ClientEvent] = Field(max_length=EVENTS_MAX)
|
||||
|
||||
|
||||
async def _parsed_events(request: Request) -> ClientEventsIn:
|
||||
async def _parsed_events(request: Request, db: Session = Depends(get_db)) -> ClientEventsIn:
|
||||
"""The body as the model — parsed only after the size cap, which is why this is a dependency and
|
||||
not a plain body parameter (FastAPI reads a declared body in full before anything else runs).
|
||||
|
||||
@@ -83,6 +83,10 @@ async def _parsed_events(request: Request) -> ClientEventsIn:
|
||||
media_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media_type != "application/json" and not media_type.endswith("+json"):
|
||||
raise HTTPException(status_code=415, detail={"code": "json_required"})
|
||||
# `current_user` resolved above on a SELECT, so this session is holding a pooled connection.
|
||||
# Give it back before awaiting the body — the sink is a background beacon and its sender may be
|
||||
# on a link far slower than the 512 KiB suggests. See `db.release_read_connection`.
|
||||
release_read_connection(db)
|
||||
raw = await capped_body(request, BODY_MAX_BYTES)
|
||||
try:
|
||||
return ClientEventsIn.model_validate_json(raw)
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.discover import hero as hero_mod
|
||||
from app.discover import pins, rails
|
||||
from app.discover.budget import Budget, DEFAULT_BUDGET
|
||||
from app.models import ComfortPin, User
|
||||
from app.plex.cards import norm_lang
|
||||
from app.routes._common import owned_or_404, not_found
|
||||
|
||||
router = APIRouter(prefix="/api/discover", tags=["discover"])
|
||||
@@ -45,6 +46,7 @@ class ReorderIn(BaseModel):
|
||||
def get_discover(
|
||||
budget: Budget = DEFAULT_BUDGET,
|
||||
seed: int = 0,
|
||||
lang: str = "en",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -52,13 +54,14 @@ def get_discover(
|
||||
|
||||
`resume` and `shelf` are built once and handed to the hero, so the two rails the hero draws from
|
||||
are not queried twice for a single page load."""
|
||||
resume = rails.continue_watching(db, user)
|
||||
lang = norm_lang(lang)
|
||||
resume = rails.continue_watching(db, user, lang=lang)
|
||||
# Plex off → the shelf rail (and so the hero's comfort tier) goes YouTube-only like the rails
|
||||
# beside it; the pins are kept and `GET /pins` still lists them (`pins.shelf`).
|
||||
shelf = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
||||
shelf = pins.shelf(db, user, include_plex=rails.plex_available(db), lang=lang)
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf),
|
||||
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf, lang=lang),
|
||||
"continue_watching": resume,
|
||||
"comfort": shelf,
|
||||
}
|
||||
@@ -80,6 +83,7 @@ def get_pick(
|
||||
seed: int = 0,
|
||||
exclude: str = "",
|
||||
start: int = 0,
|
||||
lang: str = "en",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -99,18 +103,25 @@ def get_pick(
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(
|
||||
db, user, budget, seed, exclude=set(list(skip)[:MAX_EXCLUDE]), start=start
|
||||
db,
|
||||
user,
|
||||
budget,
|
||||
seed,
|
||||
exclude=set(list(skip)[:MAX_EXCLUDE]),
|
||||
start=start,
|
||||
lang=norm_lang(lang),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/pins")
|
||||
def list_pins(
|
||||
lang: str = "en",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The shelf on its own — for the shelf editor, which needs it without the rest of the page."""
|
||||
return {"items": pins.shelf(db, user)}
|
||||
return {"items": pins.shelf(db, user, lang=norm_lang(lang))}
|
||||
|
||||
|
||||
@router.post("/pins")
|
||||
|
||||
+132
-49
@@ -9,20 +9,24 @@ serves finished files (range-aware, with the user's custom display name), and sh
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated
|
||||
from app import audit, sysconfig
|
||||
from app.auth import admin_user, require_human
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.downloads import credentials as dlcreds
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import links as linksmod
|
||||
@@ -37,8 +41,9 @@ from app.models import (
|
||||
DownloadShare,
|
||||
MediaAsset,
|
||||
User,
|
||||
display_text,
|
||||
)
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404
|
||||
from app.routes._common import capped_stream_to, get_or_404, not_found, owned_or_404
|
||||
from app.security import hash_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
@@ -398,7 +403,7 @@ def enqueue_download(
|
||||
status_code=422, detail={"code": "url_blocked", "reason": e.reason}
|
||||
) from e
|
||||
spec, profile_id = _resolve_spec(db, user, body.profile_id, body.spec)
|
||||
display_name = (body.display_name or "").strip() or None
|
||||
display_name = display_text(body.display_name)
|
||||
try:
|
||||
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
|
||||
except quota.QuotaExceeded as e:
|
||||
@@ -406,61 +411,119 @@ def enqueue_download(
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
# Read the upload in 1 MiB chunks so a large file never sits fully in memory, and the size cap is
|
||||
# enforced on the ACTUAL bytes read (a client-set Content-Length can't be trusted to gate it).
|
||||
_UPLOAD_CHUNK = 1024 * 1024
|
||||
# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
|
||||
# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
|
||||
# FastAPI *before the first dependency runs*, so `require_human` could not gate it and an anonymous
|
||||
# caller could make the process write the whole per-file allowance to temp, per request (P1; SEC-20
|
||||
# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
|
||||
# declared AFTER the auth one, is the only shape that closes it — the same one
|
||||
# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
|
||||
# every route with a raised ceiling rather than leaving it to a comment like this one.
|
||||
#
|
||||
# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a
|
||||
# Hungarian filename would otherwise be unsendable.
|
||||
UPLOAD_FILENAME_HEADER = "x-upload-filename"
|
||||
UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name"
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
def upload_download(
|
||||
file: UploadFile = File(...),
|
||||
display_name: str | None = Form(None),
|
||||
@dataclass(frozen=True)
|
||||
class SpooledUpload:
|
||||
"""What the body dependency hands the handler: the bytes already on disk, plus the two names
|
||||
that came alongside them."""
|
||||
|
||||
path: Path
|
||||
size: int
|
||||
user_id: int
|
||||
filename: str
|
||||
display_name: str | None
|
||||
|
||||
|
||||
def _header_text(request: Request, name: str) -> str:
|
||||
"""One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the
|
||||
caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's
|
||||
own fallback names the file."""
|
||||
raw = request.headers.get(name, "")
|
||||
try:
|
||||
return unquote(raw, errors="strict").strip()
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
# silent-ok: a mis-encoded header is the caller's own malformed request, and this one is
|
||||
# cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on
|
||||
# a label; logging it would hand any client a free log line per request.
|
||||
return ""
|
||||
|
||||
|
||||
async def spool_upload(
|
||||
request: Request,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Ingest a local file into the caller's library (E7 S5). Any file type is accepted; media
|
||||
(video/audio) gets the full function set, a generic file is download-only. The file is streamed
|
||||
to a temp path under the download volume (enforcing the per-file cap as it goes), then handed to
|
||||
the non-yt-dlp ingest which probes + places it and returns an already-`done` job.
|
||||
) -> AsyncIterator[SpooledUpload]:
|
||||
"""Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
|
||||
|
||||
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||
plus sync DB writes — so it must NOT run on the event loop. `file.file` is the sync file object
|
||||
Starlette spools the multipart body into."""
|
||||
`user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so
|
||||
naming `require_human` here makes the auth check a precondition of reading the body at all — a
|
||||
stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest
|
||||
MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit,
|
||||
empty, quota, probe or move error, a client that vanished mid-upload — it is removed."""
|
||||
cap = sysconfig.get_int(db, "download_upload_max_bytes")
|
||||
user_id = user.id
|
||||
# Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection`
|
||||
# for why, and for the precondition it enforces. Everything this dependency still needs is
|
||||
# already an int or a header in hand: touching an ORM attribute below would reload `user` and
|
||||
# check a connection straight back out.
|
||||
release_read_connection(db)
|
||||
staging = Path(settings.download_root) / ".staging" / "uploads"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=str(staging))
|
||||
tmp_path = Path(tmp)
|
||||
try:
|
||||
size = 0
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
while chunk := file.file.read(_UPLOAD_CHUNK):
|
||||
size += len(chunk)
|
||||
if cap and size > cap:
|
||||
raise HTTPException(
|
||||
status_code=422, detail={"code": "upload_too_large", "limit": cap}
|
||||
)
|
||||
out.write(chunk)
|
||||
if size == 0:
|
||||
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
||||
try:
|
||||
job = uploadmod.ingest_upload(
|
||||
db,
|
||||
user.id,
|
||||
filename=file.filename or "upload",
|
||||
src_path=tmp_path,
|
||||
size=size,
|
||||
display_name=(display_name or "").strip() or None,
|
||||
size = await capped_stream_to(
|
||||
request,
|
||||
out,
|
||||
cap,
|
||||
HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}),
|
||||
)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||
return _serialize_job(db, job)
|
||||
yield SpooledUpload(
|
||||
path=tmp_path,
|
||||
size=size,
|
||||
user_id=user_id,
|
||||
filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload",
|
||||
display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)),
|
||||
)
|
||||
finally:
|
||||
# Ingest MOVES the temp file into place on success (unlink is then a no-op); on any early exit
|
||||
# — cap hit, empty, quota, probe/move error — this removes the abandoned staging file.
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
def upload_download(
|
||||
upload: SpooledUpload = Depends(spool_upload),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Ingest a local file into the caller's library (E7 S5). Any file type is accepted; media
|
||||
(video/audio) gets the full function set, a generic file is download-only. The body IS the file
|
||||
(see the header note above); `spool_upload` has already authenticated the caller and streamed it
|
||||
to a temp path under the download volume, so what is left here is the non-yt-dlp ingest which
|
||||
probes + places it and returns an already-`done` job.
|
||||
|
||||
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||
plus sync DB writes — so it must NOT run on the event loop. The streaming half is async because
|
||||
reading the body is; keeping them in separate callables is what lets each run where it belongs."""
|
||||
if upload.size == 0:
|
||||
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
||||
try:
|
||||
job = uploadmod.ingest_upload(
|
||||
db,
|
||||
upload.user_id,
|
||||
filename=upload.filename,
|
||||
src_path=upload.path,
|
||||
size=upload.size,
|
||||
display_name=upload.display_name,
|
||||
)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
class EnqueueEditIn(BaseModel):
|
||||
source_job_id: int = 0
|
||||
edit_spec: dict
|
||||
@@ -480,7 +543,7 @@ def enqueue_edit(
|
||||
if source_job.status != "done" or source_job.asset_id is None:
|
||||
raise refused(409, "edit_unfinished")
|
||||
spec = body.edit_spec
|
||||
display_name = (body.display_name or "").strip() or None
|
||||
display_name = display_text(body.display_name)
|
||||
try:
|
||||
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
|
||||
except quota.QuotaExceeded as e:
|
||||
@@ -516,7 +579,20 @@ def list_downloads(
|
||||
|
||||
@router.get("/usage")
|
||||
def my_usage(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
return quota.usage(db, user.id)
|
||||
"""The caller's download quota, plus the per-file upload cap.
|
||||
|
||||
The cap rides here rather than in `quota.usage` because it is not a quota value — it is a
|
||||
`sysconfig` knob about one route — and this is the route layer, where composing the two is the
|
||||
job. The upload dialog needs it to refuse an over-cap file BEFORE sending it: both server-side
|
||||
caps answer too late to be pleasant (the edge ceiling refuses the request outright, see
|
||||
`bodylimit.ROUTE_MAX_BODY_BYTES`), and making the user push 600 MB up the wire to be told the
|
||||
limit is 500 is a bad answer even when it is a correct one. It is a convenience, not the
|
||||
boundary: the caps stay where they are and still refuse a client that ignores this.
|
||||
"""
|
||||
return {
|
||||
**quota.usage(db, user.id),
|
||||
"upload_max_bytes": sysconfig.get_int(db, "download_upload_max_bytes"),
|
||||
}
|
||||
|
||||
|
||||
# Priority when a source has several jobs (e.g. a done one plus a fresh queued one).
|
||||
@@ -611,9 +687,9 @@ def update_download_meta(
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
sent = body.model_fields_set
|
||||
if "display_name" in sent:
|
||||
job.display_name = ((body.display_name or "").strip())[:255] or None
|
||||
job.display_name = display_text(body.display_name)
|
||||
if "display_uploader" in sent:
|
||||
job.display_uploader = ((body.display_uploader or "").strip())[:255] or None
|
||||
job.display_uploader = display_text(body.display_uploader)
|
||||
if "display_uploader_url" in sent:
|
||||
job.display_uploader_url = _clean_url(body.display_uploader_url)
|
||||
if "extra_links" in sent:
|
||||
@@ -843,7 +919,14 @@ def share_download(
|
||||
user_id=recipient.id,
|
||||
type="download_shared",
|
||||
title=(job.display_name or "A download") + " was shared with you",
|
||||
data={"kind": "download_shared", "from": user.email, "job_id": job_id},
|
||||
# The client names the notice in the reader's language from `data` (E30); the title is the
|
||||
# English fallback only.
|
||||
data={
|
||||
"kind": "download_shared",
|
||||
"from": user.email,
|
||||
"job_id": job_id,
|
||||
"title": job.display_name,
|
||||
},
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
@@ -907,7 +990,7 @@ def share_recipients(
|
||||
db.execute(
|
||||
select(User)
|
||||
.where(User.id != user.id, *messageable_clauses())
|
||||
.order_by(func.lower(User.email))
|
||||
.order_by(collated(User.email, lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
|
||||
@@ -10,6 +10,7 @@ from sqlalchemy.orm.exc import StaleDataError
|
||||
from app import quota
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.mailtext import lang_for_user
|
||||
from app.feedquery import (
|
||||
decode_cursor,
|
||||
encode_cursor,
|
||||
@@ -108,7 +109,7 @@ def get_feed(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys(sort, seed, rank_expr)
|
||||
keys = sort_keys(sort, seed, rank_expr, lang_for_user(user))
|
||||
|
||||
# Expose each key column on the row so we can build the next cursor from the last item.
|
||||
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
||||
|
||||
@@ -21,6 +21,8 @@ from pydantic import BaseModel
|
||||
from sqlalchemy import and_, case, delete, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated
|
||||
from app.auth import require_human, resolved_user_id
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import Message, MessageKey, User
|
||||
@@ -246,7 +248,7 @@ def list_recipients(
|
||||
select(User)
|
||||
.join(MessageKey, MessageKey.user_id == User.id)
|
||||
.where(User.id != user.id, *messageable_clauses())
|
||||
.order_by(func.lower(func.coalesce(User.display_name, User.email)))
|
||||
.order_by(collated(func.coalesce(User.display_name, User.email), lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
|
||||
+128
-71
@@ -4,9 +4,8 @@ mirror the user's own YouTube playlists in (`/sync-youtube`) and push/revert loc
|
||||
out (`/{id}/push-plan`, `/{id}/revert-youtube`), behind the optional write scope."""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, select, and_
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app import quota
|
||||
@@ -15,14 +14,18 @@ from app.db import get_db
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
|
||||
from app.sync.playlists import (
|
||||
PlaylistGone,
|
||||
lock_playlists,
|
||||
merge_playlist,
|
||||
plan_merge,
|
||||
recompute_dirty,
|
||||
pull_user_playlists,
|
||||
repull_playlist,
|
||||
sync_user_playlists,
|
||||
sync_claim,
|
||||
user_sync_claim,
|
||||
)
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import get_or_404, owned_or_404, youtube_refusal
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
|
||||
router = APIRouter(prefix="/api/playlists", tags=["playlists"])
|
||||
@@ -79,7 +82,10 @@ def _next_playlist_position(db: Session, user_id: int) -> int:
|
||||
|
||||
|
||||
def _next_item_position(db: Session, playlist_id: int) -> int:
|
||||
"""Append position for a new item at the end of a playlist."""
|
||||
"""Append position for a new item at the end of a playlist. Locks the playlist row first: two
|
||||
adds (two tabs, the rail and the page) otherwise both read the same max and land on one
|
||||
position, and the list's order between them is then arbitrary."""
|
||||
lock_playlists(db, playlist_id)
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
|
||||
@@ -92,7 +98,10 @@ def _next_item_position(db: Session, playlist_id: int) -> int:
|
||||
|
||||
def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Append a video to a playlist if not already present (idempotent). Returns whether it
|
||||
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
||||
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later).
|
||||
Locked BEFORE the presence check: two adds of one video both saw "absent" and the second one's
|
||||
insert hit uq_playlist_video as a 500."""
|
||||
lock_playlists(db, pl.id)
|
||||
exists = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||
@@ -113,6 +122,7 @@ def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) ->
|
||||
def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Remove a video from a playlist if present (idempotent). Returns whether it was removed.
|
||||
`mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
||||
lock_playlists(db, pl.id)
|
||||
item = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||
@@ -321,13 +331,18 @@ def sync_youtube(
|
||||
Also runs automatically on the scheduler; this is the manual 'sync now'."""
|
||||
if not has_read_scope(user):
|
||||
raise _refused(403, "youtube_read_scope")
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return {"ok": True, **sync_user_playlists(db, user)}
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
with user_sync_claim(db, user.id) as free:
|
||||
if not free:
|
||||
# Another run (the scheduler, another tab) is pulling right now: "0 synced" would lie.
|
||||
raise _refused(409, "sync_all_busy")
|
||||
# After the claim, like the re-pull's: a refused attempt must not spend one of the pulls.
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return {"ok": True, **pull_user_playlists(db, user)}
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
@router.post("/{playlist_id}/revert-youtube")
|
||||
@@ -344,13 +359,20 @@ def revert_youtube(
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_read_scope(user):
|
||||
raise _refused(403, "youtube_read_scope")
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return {"ok": True, **repull_playlist(db, user, pl)}
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
raise _refused(409, "sync_busy")
|
||||
# The cooldown is spent only by a pull that runs: checked after the claim, so clicking
|
||||
# Revert while a push is in flight does not use up the three pulls per ten minutes.
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return {"ok": True, **repull_playlist(db, user, pl)}
|
||||
except PlaylistGone:
|
||||
raise not_found("playlist")
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
@@ -392,25 +414,32 @@ def push(
|
||||
neither side loses an item). Refuses if the estimated quota exceeds what's left today, so a
|
||||
big sync can't strand half-applied. Write scope required (require_write_scope)."""
|
||||
pl = _pushable(db, user, playlist_id)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
||||
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||
live = None
|
||||
if pl.yt_playlist_id:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
live = (
|
||||
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||
)
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise _refused(429, "quota_exceeded")
|
||||
return {"ok": True, **merge_playlist(db, user, pl, live=live)}
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
# One sync of a list at a time, across tabs, devices and the scheduler (`sync_claim`): the
|
||||
# client's own in-flight flag only covers the menus of one page.
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
raise _refused(409, "sync_busy")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
||||
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||
live = None
|
||||
if pl.yt_playlist_id:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
live = (
|
||||
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||
)
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise _refused(429, "quota_exceeded")
|
||||
return {"ok": True, **merge_playlist(db, user, pl, live=live)}
|
||||
except PlaylistGone:
|
||||
raise not_found("playlist")
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
@router.get("/{playlist_id}")
|
||||
@@ -427,7 +456,8 @@ def get_playlist(
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id))
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
.order_by(PlaylistItem.position)
|
||||
# `id` breaks a position tie the same way on every read (the renumbering add does too).
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).all()
|
||||
return {
|
||||
"id": pl.id,
|
||||
@@ -451,7 +481,8 @@ def rename_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
# Locked: the new name moves `dirty`, which the scheduled mirror refresh reads under this lock.
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||
if "name" in body.model_fields_set:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
@@ -491,33 +522,65 @@ def delete_playlist(
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if pl.kind == "watch_later":
|
||||
raise _refused(400, "watch_later_undeletable")
|
||||
if on_youtube:
|
||||
if not pl.yt_playlist_id:
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_write_scope(user):
|
||||
raise _refused(403, "youtube_editing_off")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt.delete_playlist(pl.yt_playlist_id)
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc)
|
||||
db.delete(pl) # items cascade
|
||||
db.commit()
|
||||
# Not in the middle of a sync of it (`sync_claim`): a push that read the list before the delete
|
||||
# and wrote YouTube after it planned from no items at all, so it emptied the YouTube playlist a
|
||||
# local-only delete means to keep. The push checks for a vanished list too (`PlaylistGone`);
|
||||
# this refuses the delete while a sync runs rather than race it.
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
raise _refused(409, "sync_busy")
|
||||
if on_youtube:
|
||||
if not pl.yt_playlist_id:
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_write_scope(user):
|
||||
raise _refused(403, "youtube_editing_off")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt.delete_playlist(pl.yt_playlist_id)
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc)
|
||||
# The playlist first, as every item write: the cascade deletes the items before the list row.
|
||||
lock_playlists(db, pl.id)
|
||||
db.delete(pl) # items cascade
|
||||
db.commit()
|
||||
return {"deleted": playlist_id}
|
||||
|
||||
|
||||
class AddItemIn(VideoRefIn):
|
||||
# Where the video lands among the list's items, in their stored order (0 = first). Omitted, it
|
||||
# is appended. The list page's "Undo" of a removal puts the row back where it was with this —
|
||||
# ONE request, because append-then-reorder leaves it at the end whenever the second call fails.
|
||||
position: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
@router.post("/{playlist_id}/items")
|
||||
def add_item(
|
||||
playlist_id: int,
|
||||
body: VideoRefIn,
|
||||
body: AddItemIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
video_id = body.video_id
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
_add_item(db, pl, video_id, mark_dirty=True)
|
||||
added = _add_item(db, pl, video_id, mark_dirty=False)
|
||||
if added and body.position is not None:
|
||||
# Renumber the whole list with the new row at `position` (clamped to the end): positions
|
||||
# can have gaps after removals, so "index N" is only meaningful over the ordered rows.
|
||||
rows = list(
|
||||
db.execute(
|
||||
select(PlaylistItem)
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).scalars()
|
||||
)
|
||||
new = next(r for r in rows if r.video_id == video_id)
|
||||
rows.remove(new)
|
||||
rows.insert(min(body.position, len(rows)), new)
|
||||
for pos, row in enumerate(rows):
|
||||
row.position = pos
|
||||
recompute_dirty(db, pl)
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
@@ -562,6 +625,8 @@ def move_item(
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "playlist_move", "reason": "same_playlist"}
|
||||
)
|
||||
# Both lists, in id order: a move A→B beside a move B→A would otherwise lock crosswise.
|
||||
lock_playlists(db, src.id, dst.id)
|
||||
item = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == src.id, PlaylistItem.video_id == video_id
|
||||
@@ -581,20 +646,11 @@ def move_item(
|
||||
if already is None:
|
||||
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
|
||||
# survives the move, and there is no window where the video belongs to neither list. The
|
||||
# check above is not a lock: a second tab's add or the scheduler's YouTube merge can insert
|
||||
# the same (playlist, video) in between, and uq_playlist_video then refuses the re-parent.
|
||||
# That is the already-present outcome arriving late, not a failure — answer it the same way
|
||||
# instead of a raw 500.
|
||||
try:
|
||||
with db.begin_nested():
|
||||
item.playlist_id = dst.id
|
||||
item.position = _next_item_position(db, dst.id)
|
||||
db.flush()
|
||||
dst_changed = True
|
||||
# silent-ok: the conflict IS the answer — the target already holds the video, which is the
|
||||
# idempotent outcome; the savepoint put the row back in the source list, dropped below.
|
||||
except IntegrityError:
|
||||
pass
|
||||
# check above holds: both lists are locked, and an insert into the target (another tab's
|
||||
# add, a YouTube merge) waits for the lock — its foreign key needs the target row.
|
||||
item.playlist_id = dst.id
|
||||
item.position = _next_item_position(db, dst.id)
|
||||
dst_changed = True
|
||||
if not dst_changed:
|
||||
db.delete(item)
|
||||
# The source always changed; the target only when the row landed in it. `_touch` on a target
|
||||
@@ -619,7 +675,8 @@ def reorder_items(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
# Locked: a concurrent add between reading the items and writing the order would collide.
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||
order = body.video_ids
|
||||
items = {
|
||||
it.video_id: it
|
||||
|
||||
@@ -4,6 +4,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import PlexItem
|
||||
from app.plex.cards import norm_lang as _norm_lang # noqa: F401 — the package's historic name
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.routes._common import not_found
|
||||
|
||||
@@ -15,8 +16,3 @@ def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
||||
if it is None:
|
||||
raise not_found("plex_item")
|
||||
return it
|
||||
|
||||
|
||||
def _norm_lang(lang: str | None) -> str:
|
||||
"""The overlay has an EN base + an optional HU twin; anything other than `hu` serves EN."""
|
||||
return "hu" if lang == "hu" else "en"
|
||||
|
||||
@@ -10,10 +10,13 @@ from sqlalchemy import (
|
||||
func,
|
||||
literal,
|
||||
null,
|
||||
or_,
|
||||
select,
|
||||
)
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated, sorted_words
|
||||
from app import sysconfig
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
@@ -25,7 +28,7 @@ from app.models import (
|
||||
User,
|
||||
)
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import episode_card
|
||||
from app.plex.cards import display_title, episode_card
|
||||
from app.plex.query import (
|
||||
FIELD_COLUMNS,
|
||||
PLAYBACK_CLASSES,
|
||||
@@ -54,14 +57,15 @@ from app.utils import to_tsquery_str
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
|
||||
def sort_column(u, sort: str):
|
||||
def sort_column(u, sort: str, lang: str = "en"):
|
||||
"""The browse grid's sort key -> orderable column on the union subquery. Module-level (not inline
|
||||
in the handler) so a test can pin its vocabulary to `PLEX_SORT_KEYS` — a key the Sort control
|
||||
offers but this cannot order would quietly fall back to `added_at` and look like a broken control.
|
||||
An unknown key (a stale bookmark, a hand-edited URL) deliberately keeps that fallback."""
|
||||
An unknown key (a stale bookmark, a hand-edited URL) deliberately keeps that fallback. The title sorts
|
||||
in the reader's alphabet (`collation`)."""
|
||||
return {
|
||||
"added": u.c.added_at,
|
||||
"title": func.lower(u.c.title),
|
||||
"title": collated(u.c.title, lang),
|
||||
"year": u.c.year,
|
||||
"rating": u.c.rating,
|
||||
"duration": u.c.duration_s,
|
||||
@@ -78,7 +82,7 @@ def list_libraries(user: User = Depends(current_user), db: Session = Depends(get
|
||||
db.execute(
|
||||
select(PlexLibrary)
|
||||
.where(PlexLibrary.overlay_of.is_(None))
|
||||
.order_by(PlexLibrary.kind.desc(), PlexLibrary.title)
|
||||
.order_by(PlexLibrary.kind.desc(), collated(PlexLibrary.title, lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -237,7 +241,7 @@ def unified_library(
|
||||
if tsq is not None:
|
||||
order = [u.c.rank.desc()]
|
||||
else:
|
||||
col = sort_column(u, sort)
|
||||
col = sort_column(u, sort, lang)
|
||||
order = [(col.asc() if sort_dir == "asc" else col.desc()).nullslast()]
|
||||
order.append(u.c.rk.desc())
|
||||
rows = db.execute(select(u).order_by(*order).offset(offset).limit(limit)).all()
|
||||
@@ -281,26 +285,32 @@ def unified_library(
|
||||
episodes = []
|
||||
if tsq is not None and want_shows and field in ("all", "episode"):
|
||||
ep_st = aliased(PlexState)
|
||||
ep_vec = (
|
||||
func.to_tsvector(TS_CONFIG, func.coalesce(PlexItem.title, ""))
|
||||
if field == "episode"
|
||||
else PlexItem.search_vector
|
||||
)
|
||||
# In HU an episode is found by its Hungarian title too (E30: the overlay now fills it) — and
|
||||
# still by its English one, like `vector_match` does for the grid.
|
||||
if field == "episode":
|
||||
ep_vec = func.to_tsvector(TS_CONFIG, func.coalesce(PlexItem.title, ""))
|
||||
ep_vec_hu = func.to_tsvector(TS_CONFIG, func.coalesce(PlexItem.title_hu, ""))
|
||||
else:
|
||||
ep_vec, ep_vec_hu = PlexItem.search_vector, PlexItem.search_vector_hu
|
||||
hit = ep_vec.op("@@")(tsq)
|
||||
if lang == "hu":
|
||||
hit = or_(ep_vec_hu.op("@@")(tsq), hit)
|
||||
ep_vec = ep_vec_hu
|
||||
eps = db.execute(
|
||||
select(PlexItem, ep_st, PlexShow.title.label("show_title"))
|
||||
select(PlexItem, ep_st, PlexShow)
|
||||
.outerjoin(ep_st, and_(ep_st.item_id == PlexItem.id, ep_st.user_id == user.id))
|
||||
.outerjoin(PlexShow, PlexShow.id == PlexItem.show_id)
|
||||
.where(
|
||||
PlexItem.kind == "episode",
|
||||
PlexItem.library_id.in_(show_lib_ids),
|
||||
ep_vec.op("@@")(tsq),
|
||||
hit,
|
||||
)
|
||||
.order_by(func.ts_rank(ep_vec, tsq).desc())
|
||||
.limit(24)
|
||||
).all()
|
||||
for e, s, show_title in eps:
|
||||
card = episode_card(e, s)
|
||||
card["show_title"] = show_title
|
||||
for e, s, sh in eps:
|
||||
card = episode_card(e, s, lang=lang)
|
||||
card["show_title"] = display_title(sh, lang) if sh is not None else None
|
||||
episodes.append(card)
|
||||
|
||||
return {
|
||||
@@ -523,16 +533,19 @@ def facets(
|
||||
for c in split_csv(f.playable):
|
||||
if c in PLAYBACK_CLASSES:
|
||||
playable_counts.setdefault(c, 0)
|
||||
# Names in the reader's alphabet: the genres are Hungarian words for a Hungarian reader (E30 S1).
|
||||
alpha = {w: i for i, w in enumerate(sorted_words(db, [*genre_counts, *studio_counts], lang))}
|
||||
return {
|
||||
"genres": [
|
||||
{"value": g, "count": c} for g, c in sorted(genre_counts.items(), key=lambda kv: kv[0])
|
||||
{"value": g, "count": c}
|
||||
for g, c in sorted(genre_counts.items(), key=lambda kv: alpha[kv[0]])
|
||||
],
|
||||
"content_ratings": [
|
||||
{"value": cr, "count": c} for cr, c in sorted(cr_counts.items(), key=lambda kv: -kv[1])
|
||||
],
|
||||
"studios": [
|
||||
{"value": s, "count": c}
|
||||
for s, c in sorted(studio_counts.items(), key=lambda kv: (-kv[1], kv[0]))
|
||||
for s, c in sorted(studio_counts.items(), key=lambda kv: (-kv[1], alpha[kv[0]]))
|
||||
],
|
||||
# Fixed good→heavy order (not by count), so the class list reads as a severity scale.
|
||||
"playable": [
|
||||
|
||||
@@ -9,6 +9,8 @@ from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated, sorted_words
|
||||
from app import logs
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
@@ -22,10 +24,10 @@ from app.models import (
|
||||
)
|
||||
from app.plex import rollup
|
||||
from app.plex import sync as plex_sync
|
||||
from app.plex.cards import movie_card, show_card
|
||||
from app.plex.cards import display_title, movie_card, norm_lang, show_card
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.plex.query import content_libs, runtime_by_key, top_art_keys
|
||||
from app.plex.query import content_libs, runtime_by_key, title_col, top_art_keys
|
||||
from app.utils import like_escape
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
@@ -142,7 +144,7 @@ def collections(
|
||||
db.execute(
|
||||
query.order_by(
|
||||
PlexCollection.child_count.desc().nullslast(),
|
||||
func.lower(PlexCollection.title),
|
||||
collated(PlexCollection.title, lang_for_user(user)),
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
@@ -208,16 +210,24 @@ MEMBER_SORT_KEYS: dict[str, Callable[[Member], Any]] = {
|
||||
DEFAULT_MEMBER_SORT = "year"
|
||||
|
||||
|
||||
def sort_members(members: list[Member], sort: str, sort_dir: str) -> list[Member]:
|
||||
def sort_members(
|
||||
members: list[Member], sort: str, sort_dir: str, alpha: dict[str, int] | None = None
|
||||
) -> list[Member]:
|
||||
"""Order a collection's members by one of `PLEX_SORT_KEYS`.
|
||||
|
||||
Two properties worth stating because they are easy to get wrong with a plain `reverse=True`:
|
||||
members with no value for the key sink to the END in BOTH directions (matching the browse grid's
|
||||
`nullslast()` — a movie with no rating is not "the worst rated"), and ties always resolve
|
||||
title-ascending, so flipping the direction reverses the ranking rather than the alphabet inside
|
||||
it. An unknown key falls back to the page's default instead of returning an arbitrary order."""
|
||||
it. An unknown key falls back to the page's default instead of returning an arbitrary order.
|
||||
|
||||
`alpha` ranks each title in the reader's alphabet (`collation.sorted_words`); without it, titles
|
||||
compare as Python strings, which puts every accented initial after "z"."""
|
||||
by_title = (lambda m: alpha[m.title]) if alpha is not None else (lambda m: m.title)
|
||||
key = MEMBER_SORT_KEYS.get(sort, MEMBER_SORT_KEYS[DEFAULT_MEMBER_SORT])
|
||||
ordered = sorted(members, key=lambda m: m.title) # stable secondary, applied first
|
||||
if key is MEMBER_SORT_KEYS["title"]:
|
||||
key = by_title
|
||||
ordered = sorted(members, key=by_title) # stable secondary, applied first
|
||||
have = [m for m in ordered if key(m) is not None]
|
||||
missing = [m for m in ordered if key(m) is None]
|
||||
have.sort(key=key, reverse=sort_dir != "asc")
|
||||
@@ -229,6 +239,7 @@ def collection_detail(
|
||||
rating_key: str,
|
||||
sort: str = DEFAULT_MEMBER_SORT,
|
||||
sort_dir: str = "asc",
|
||||
lang: str = "en",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -237,6 +248,7 @@ def collection_detail(
|
||||
control means the same thing on both surfaces; the default is release year ascending, which is
|
||||
what this page did before it had a control. Read-only here; edits go through the admin mutation
|
||||
endpoints below."""
|
||||
lang = norm_lang(lang)
|
||||
col = by_rating_key(db, PlexCollection, rating_key)
|
||||
if col is None:
|
||||
raise not_found("collection")
|
||||
@@ -244,7 +256,7 @@ def collection_detail(
|
||||
db.execute(
|
||||
select(PlexItem)
|
||||
.where(PlexItem.kind == "movie", PlexItem.collection_keys.contains([rating_key]))
|
||||
.order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title))
|
||||
.order_by(PlexItem.year.asc().nullslast(), collated(title_col(PlexItem, lang), lang))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -253,7 +265,7 @@ def collection_detail(
|
||||
db.execute(
|
||||
select(PlexShow)
|
||||
.where(PlexShow.collection_keys.contains([rating_key]))
|
||||
.order_by(PlexShow.year.asc().nullslast(), func.lower(PlexShow.title))
|
||||
.order_by(PlexShow.year.asc().nullslast(), collated(title_col(PlexShow, lang), lang))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -285,7 +297,7 @@ def collection_detail(
|
||||
Member(
|
||||
kind="movie",
|
||||
obj=m,
|
||||
title=(m.title or "").lower(),
|
||||
title=display_title(m, lang).lower(),
|
||||
year=m.year,
|
||||
rating=m.rating,
|
||||
added=m.added_at,
|
||||
@@ -298,7 +310,7 @@ def collection_detail(
|
||||
Member(
|
||||
kind="show",
|
||||
obj=sh,
|
||||
title=(sh.title or "").lower(),
|
||||
title=display_title(sh, lang).lower(),
|
||||
year=sh.year,
|
||||
rating=sh.rating,
|
||||
added=sh.added_at,
|
||||
@@ -308,11 +320,12 @@ def collection_detail(
|
||||
)
|
||||
for sh in shows
|
||||
]
|
||||
members = sort_members(members, sort, sort_dir)
|
||||
alpha = {w: i for i, w in enumerate(sorted_words(db, [m.title for m in members], lang))}
|
||||
members = sort_members(members, sort, sort_dir, alpha)
|
||||
items = [
|
||||
movie_card(m.obj, mstate.get(m.obj.id))
|
||||
movie_card(m.obj, mstate.get(m.obj.id), lang=lang)
|
||||
if m.kind == "movie"
|
||||
else show_card(m.obj, sstate.get(m.obj.id, "new"))
|
||||
else show_card(m.obj, sstate.get(m.obj.id, "new"), lang=lang)
|
||||
for m in members
|
||||
]
|
||||
years = [m.year for m in members if m.year is not None]
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
"""Plex show/item detail: the show drill-down + the per-item player payload (markers, cast, nav)."""
|
||||
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.collation import collated, sorted_words
|
||||
from app import logs
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
@@ -24,6 +26,7 @@ from app.models import (
|
||||
from app.plex import detail as plex_detail
|
||||
from app.plex import people as plex_people
|
||||
from app.plex import rollup, subtitles
|
||||
from app.plex.query import title_col
|
||||
from app.plex import tmdb as plex_tmdb
|
||||
from app.plex.cards import episode_card, movie_card, show_card
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
@@ -107,7 +110,8 @@ def show_detail(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""A show's seasons + episodes (the drill-down view) with per-user watch state."""
|
||||
hu = _norm_lang(lang) == "hu"
|
||||
lang = _norm_lang(lang)
|
||||
hu = lang == "hu"
|
||||
sh = by_rating_key(db, PlexShow, rating_key)
|
||||
if sh is None:
|
||||
raise not_found("show")
|
||||
@@ -144,7 +148,7 @@ def show_detail(
|
||||
by_season: dict[int | None, list] = {}
|
||||
all_cards: list[dict] = [] # every episode, in season/episode order → show-level rollup
|
||||
for e in eps:
|
||||
card = episode_card(e, states.get(e.id))
|
||||
card = episode_card(e, states.get(e.id), lang=lang)
|
||||
by_season.setdefault(e.season_id, []).append(card)
|
||||
all_cards.append(card)
|
||||
show_roll = rollup.rollup(all_cards)
|
||||
@@ -156,7 +160,7 @@ def show_detail(
|
||||
related: list[dict] = []
|
||||
try:
|
||||
meta, related_raw = _show_enrichment(db, sh.rating_key)
|
||||
related = _relatedshow_cards(db, user.id, related_raw, exclude=sh.id)
|
||||
related = _relatedshow_cards(db, user.id, related_raw, exclude=sh.id, lang=lang)
|
||||
if sh.people_enriched_at is None and meta:
|
||||
plex_people.enrich_row_people(db, sh, plex_people.cast_from_meta(meta))
|
||||
plex_detail.apply_rich_detail(sh, meta) # imdb/tagline/full_cast for future views
|
||||
@@ -210,7 +214,7 @@ def show_detail(
|
||||
"episode_count": show_roll["episode_count"],
|
||||
"collection_keys": sh.collection_keys or [],
|
||||
# TMDB enrichment (E5 S2), served from the row.
|
||||
"tmdb": plex_tmdb.serve_block(db, sh),
|
||||
"tmdb": plex_tmdb.serve_block(db, sh, lang=lang),
|
||||
},
|
||||
"seasons": [_season_block(se, by_season.get(se.id, [])) for se in seasons],
|
||||
"related": related,
|
||||
@@ -234,7 +238,9 @@ def _season_block(se: PlexSeason, cards: list[dict]) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _relatedshow_cards(db: Session, user_id: int, related: list[dict], exclude: int) -> list[dict]:
|
||||
def _relatedshow_cards(
|
||||
db: Session, user_id: int, related: list[dict], exclude: int, lang: str
|
||||
) -> list[dict]:
|
||||
"""Map Plex 'related' metadata to show cards for the shows we actually mirror (so they're openable),
|
||||
excluding the current show. Order preserved; capped."""
|
||||
rks = [str(m.get("ratingKey")) for m in related if m.get("ratingKey")]
|
||||
@@ -254,13 +260,13 @@ def _relatedshow_cards(db: Session, user_id: int, related: list[dict], exclude:
|
||||
if sh is None or rk in seen:
|
||||
continue
|
||||
seen.add(rk)
|
||||
out.append(show_card(sh, statuses.get(sh.id, "new")))
|
||||
out.append(show_card(sh, statuses.get(sh.id, "new"), lang=lang))
|
||||
if len(out) >= 20:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _collection_strips(db: Session, it: PlexItem, user_id: int) -> list[dict]:
|
||||
def _collection_strips(db: Session, it: PlexItem, user_id: int, lang: str) -> list[dict]:
|
||||
"""The collections this movie belongs to, each with its member titles (playable cards) — the
|
||||
'Avatar Collection' strip on the info page. Pure local lookup (no Plex call)."""
|
||||
keys = it.collection_keys or []
|
||||
@@ -273,7 +279,8 @@ def _collection_strips(db: Session, it: PlexItem, user_id: int) -> list[dict]:
|
||||
).scalars()
|
||||
}
|
||||
# Smallest (most specific — e.g. a franchise) first; the big auto-lists (IMDb/TMDb Popular) last.
|
||||
ordered = sorted((c for c in cols.values()), key=lambda c: (c.child_count or 0, c.title))
|
||||
alpha = {w: i for i, w in enumerate(sorted_words(db, [c.title for c in cols.values()], lang))}
|
||||
ordered = sorted(cols.values(), key=lambda c: (c.child_count or 0, alpha[c.title]))
|
||||
out = []
|
||||
for col in ordered:
|
||||
rk = col.rating_key
|
||||
@@ -285,7 +292,9 @@ def _collection_strips(db: Session, it: PlexItem, user_id: int) -> list[dict]:
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.collection_keys.contains([rk]),
|
||||
)
|
||||
.order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title))
|
||||
.order_by(
|
||||
PlexItem.year.asc().nullslast(), collated(title_col(PlexItem, lang), lang)
|
||||
)
|
||||
.limit(60)
|
||||
)
|
||||
.scalars()
|
||||
@@ -305,7 +314,7 @@ def _collection_strips(db: Session, it: PlexItem, user_id: int) -> list[dict]:
|
||||
"id": rk,
|
||||
"title": col.title,
|
||||
"source": _collection_source(col),
|
||||
"items": [movie_card(m, states.get(m.id)) for m in members],
|
||||
"items": [movie_card(m, states.get(m.id), lang=lang) for m in members],
|
||||
}
|
||||
)
|
||||
return out
|
||||
@@ -339,6 +348,26 @@ def _episode_nav(db: Session, it: PlexItem) -> tuple[str | None, str | None]:
|
||||
return (keys[i - 1] if i > 0 else None), (keys[i + 1] if i < len(keys) - 1 else None)
|
||||
|
||||
|
||||
# The technical tail of Plex's track title — "English (AC3 5.1)" → "AC3 5.1".
|
||||
_TRACK_TAIL = re.compile(r"\(([^()]+)\)\s*$")
|
||||
|
||||
|
||||
def _track_naming(s: dict) -> dict:
|
||||
"""What the player names a track from (E30). Plex's `displayTitle` is in the SERVER's language
|
||||
("English (AC3 5.1)", "magyar Forced", "中文 (EAC3 Stereo)"), so it is sent only as the fallback
|
||||
`label`; the client builds the name in the reader's language from the language tag, the
|
||||
forced / SDH flags and the codec tail. No title and no language → no label (the client numbers it)."""
|
||||
title = s.get("displayTitle") or s.get("language") or None
|
||||
tail = _TRACK_TAIL.search(title) if title else None
|
||||
return {
|
||||
"label": title,
|
||||
"lang_tag": s.get("languageTag") or None,
|
||||
"forced": bool(s.get("forced")),
|
||||
"hearing_impaired": bool(s.get("hearingImpaired")),
|
||||
"detail": tail.group(1).strip() if tail else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/item/{rating_key}")
|
||||
def item_detail(
|
||||
rating_key: str,
|
||||
@@ -384,9 +413,7 @@ def item_detail(
|
||||
audio_streams.append(
|
||||
{
|
||||
"ord": len(audio_streams),
|
||||
"label": s.get("displayTitle")
|
||||
or s.get("language")
|
||||
or f"Audio {len(audio_streams) + 1}",
|
||||
**_track_naming(s),
|
||||
"language": s.get("languageTag") or s.get("language"),
|
||||
"default": bool(s.get("selected") or s.get("default")),
|
||||
}
|
||||
@@ -396,9 +423,7 @@ def item_detail(
|
||||
subtitle_streams.append(
|
||||
{
|
||||
"ord": len(subtitle_streams),
|
||||
"label": s.get("displayTitle")
|
||||
or s.get("language")
|
||||
or f"Subtitle {len(subtitle_streams) + 1}",
|
||||
**_track_naming(s),
|
||||
"language": s.get("languageTag") or s.get("language"),
|
||||
"codec": codec,
|
||||
# Text subs (srt/ass/…) can be served as WebVTT tracks; image subs (PGS/VobSub)
|
||||
@@ -468,7 +493,7 @@ def item_detail(
|
||||
"genres": (it.genres_hu or it.genres if hu else it.genres) or [],
|
||||
"directors": it.directors or [],
|
||||
"studio": it.studio,
|
||||
"tagline": it.tagline,
|
||||
"tagline": (it.tagline_hu or it.tagline) if hu else it.tagline,
|
||||
}
|
||||
else:
|
||||
rich = plex_detail.rich_detail(meta)
|
||||
@@ -512,7 +537,7 @@ def item_detail(
|
||||
"episode_number": it.episode_number,
|
||||
"prev_id": prev_id,
|
||||
"next_id": next_id,
|
||||
"collections": _collection_strips(db, it, user.id),
|
||||
"collections": _collection_strips(db, it, user.id, lang),
|
||||
# TMDB enrichment (E5 S2), served from the row — movies only (episodes aren't enriched).
|
||||
"tmdb": plex_tmdb.serve_block(db, it) if it.kind == "movie" else None,
|
||||
"tmdb": plex_tmdb.serve_block(db, it, lang=lang) if it.kind == "movie" else None,
|
||||
}
|
||||
|
||||
@@ -18,11 +18,12 @@ from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.collation import sorted_words
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import PlexItem, PlexShow, PlexState, User
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, movie_card, show_card
|
||||
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||
from app.plex.query import (
|
||||
LibraryFilters,
|
||||
apply_meta_filters,
|
||||
@@ -61,14 +62,14 @@ def _movie_meta(q, p: dict, lang: str, f: LibraryFilters):
|
||||
return q
|
||||
|
||||
|
||||
def _movie_cards(db: Session, uid: int, items: list[PlexItem]) -> list[dict]:
|
||||
def _movie_cards(db: Session, uid: int, items: list[PlexItem], lang: str) -> list[dict]:
|
||||
sts = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||
return [movie_card(it, sts.get(it.id)) for it in items]
|
||||
return [movie_card(it, sts.get(it.id), lang=lang) for it in items]
|
||||
|
||||
|
||||
def _show_cards(db: Session, uid: int, shows: list[PlexShow]) -> list[dict]:
|
||||
def _show_cards(db: Session, uid: int, shows: list[PlexShow], lang: str) -> list[dict]:
|
||||
statuses = rollup.state_map(db, uid, [s.id for s in shows])
|
||||
return [show_card(s, statuses.get(s.id, "new")) for s in shows]
|
||||
return [show_card(s, statuses.get(s.id, "new"), lang=lang) for s in shows]
|
||||
|
||||
|
||||
def _recently_added(
|
||||
@@ -123,8 +124,8 @@ def _recently_added(
|
||||
# share one), so the merge doesn't depend on object identity staying alive.
|
||||
top_movies = [o for _, kind, o in top if kind == "movie"]
|
||||
top_shows = [o for _, kind, o in top if kind == "show"]
|
||||
mcards = {m.rating_key: c for m, c in zip(top_movies, _movie_cards(db, uid, top_movies))}
|
||||
scards = {s.rating_key: c for s, c in zip(top_shows, _show_cards(db, uid, top_shows))}
|
||||
mcards = {m.rating_key: c for m, c in zip(top_movies, _movie_cards(db, uid, top_movies, lang))}
|
||||
scards = {s.rating_key: c for s, c in zip(top_shows, _show_cards(db, uid, top_shows, lang))}
|
||||
return [mcards[o.rating_key] if kind == "movie" else scards[o.rating_key] for _, kind, o in top]
|
||||
|
||||
|
||||
@@ -156,10 +157,10 @@ def _recently_watched(
|
||||
.order_by(PlexState.watched_at.desc().nullslast())
|
||||
.limit(RAIL_LIMIT)
|
||||
).all():
|
||||
rows.append((st.watched_at, leaf_card(it, st)))
|
||||
rows.append((st.watched_at, leaf_card(it, st, lang=lang)))
|
||||
if show_ids:
|
||||
eq = (
|
||||
select(PlexItem, PlexState, PlexShow.title, PlexShow.rating_key)
|
||||
select(PlexItem, PlexState, PlexShow)
|
||||
.join(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
|
||||
.join(PlexShow, PlexShow.id == PlexItem.show_id)
|
||||
.where(
|
||||
@@ -170,8 +171,11 @@ def _recently_watched(
|
||||
.order_by(PlexState.watched_at.desc().nullslast())
|
||||
.limit(RAIL_LIMIT)
|
||||
)
|
||||
for ep, st, sh_title, sh_rk in db.execute(eq).all():
|
||||
rows.append((st.watched_at, leaf_card(ep, st, show_title=sh_title, show_rk=sh_rk)))
|
||||
for ep, st, sh in db.execute(eq).all():
|
||||
card = leaf_card(
|
||||
ep, st, show_title=display_title(sh, lang), show_rk=sh.rating_key, lang=lang
|
||||
)
|
||||
rows.append((st.watched_at, card))
|
||||
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
||||
return [card for _, card in rows[:RAIL_LIMIT]]
|
||||
|
||||
@@ -214,7 +218,7 @@ def _continue_watching(
|
||||
.limit(RAIL_LIMIT)
|
||||
).all()
|
||||
for it, st in mrows:
|
||||
rows.append((st.progress_updated_at, movie_card(it, st)))
|
||||
rows.append((st.progress_updated_at, movie_card(it, st, lang=lang)))
|
||||
|
||||
if show_ids:
|
||||
# Reuse the shared per-show episode rollup (total/watched/inprog) rather than re-inlining it, plus
|
||||
@@ -234,7 +238,7 @@ def _continue_watching(
|
||||
.limit(RAIL_LIMIT)
|
||||
).all()
|
||||
for sh, resumed_at in srows:
|
||||
rows.append((resumed_at, show_card(sh, "in_progress")))
|
||||
rows.append((resumed_at, show_card(sh, "in_progress", lang=lang)))
|
||||
|
||||
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
||||
return [card for _, card in rows[:RAIL_LIMIT]]
|
||||
@@ -261,7 +265,8 @@ def _genre_tiles(
|
||||
expanded = base.subquery()
|
||||
for g, c in db.execute(select(expanded.c.g, func.count()).group_by(expanded.c.g)).all():
|
||||
counts[g] = counts.get(g, 0) + c
|
||||
top = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[:GENRE_TILES]
|
||||
alpha = {w: i for i, w in enumerate(sorted_words(db, list(counts), lang))}
|
||||
top = sorted(counts.items(), key=lambda kv: (-kv[1], alpha[kv[0]]))[:GENRE_TILES]
|
||||
# A 2×2 poster collage per tile: the top movies in that genre (color hash is the client fallback).
|
||||
art = top_art_keys(
|
||||
db,
|
||||
@@ -310,7 +315,7 @@ def _because_you_watched(
|
||||
)
|
||||
if anchor is None:
|
||||
return None
|
||||
resolved = resolve_recs(db, anchor.recommendations)
|
||||
resolved = resolve_recs(db, anchor.recommendations, lang=lang)
|
||||
# Preserve TMDB's recommendation order, keep only in-library hits, and drop the anchor itself.
|
||||
rks = [
|
||||
r["rating_key"]
|
||||
@@ -346,9 +351,11 @@ def _because_you_watched(
|
||||
ordered_shows = [shows[rk] for rk in rks if rk in shows]
|
||||
# rating_key is globally unique in Plex, so it keys the serialized cards back safely (no id()).
|
||||
mcards = {
|
||||
m.rating_key: c for m, c in zip(ordered_movies, _movie_cards(db, uid, ordered_movies))
|
||||
m.rating_key: c for m, c in zip(ordered_movies, _movie_cards(db, uid, ordered_movies, lang))
|
||||
}
|
||||
scards = {
|
||||
s.rating_key: c for s, c in zip(ordered_shows, _show_cards(db, uid, ordered_shows, lang))
|
||||
}
|
||||
scards = {s.rating_key: c for s, c in zip(ordered_shows, _show_cards(db, uid, ordered_shows))}
|
||||
items: list[dict] = []
|
||||
for rk in rks:
|
||||
if rk in mcards:
|
||||
@@ -357,7 +364,10 @@ def _because_you_watched(
|
||||
items.append(scards[rk])
|
||||
if not items:
|
||||
return None
|
||||
return {"anchor": {"id": anchor.rating_key, "title": anchor.title}, "items": items}
|
||||
return {
|
||||
"anchor": {"id": anchor.rating_key, "title": display_title(anchor, lang)},
|
||||
"items": items,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/discover")
|
||||
@@ -456,8 +466,8 @@ def surprise(
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
mcards = _movie_cards(db, user.id, movies)
|
||||
scards = _show_cards(db, user.id, shows)
|
||||
mcards = _movie_cards(db, user.id, movies, lang)
|
||||
scards = _show_cards(db, user.id, shows, lang)
|
||||
# Interleave movie/show cards, then trim to `limit`.
|
||||
items: list[dict] = []
|
||||
for i in range(max(len(mcards), len(scards))):
|
||||
|
||||
@@ -7,7 +7,8 @@ from fastapi import APIRouter, BackgroundTasks, Depends
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.collation import collated
|
||||
from app import logs, sysconfig
|
||||
from app.auth import current_user
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import (
|
||||
@@ -18,22 +19,28 @@ from app.models import (
|
||||
)
|
||||
from app.plex import rollup
|
||||
from app.plex import tmdb as plex_tmdb
|
||||
from app.plex.cards import movie_card, show_card
|
||||
from app.plex.query import content_libs, split_csv
|
||||
from app.plex.cards import movie_card, norm_lang, show_card
|
||||
from app.plex.query import content_libs, split_csv, title_col
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
# How many "Known for" titles the person page shows (the stored pool is deeper; see plex.tmdb).
|
||||
_KNOWN_FOR_SHOWN = 15
|
||||
|
||||
|
||||
def _resolve_known_for(
|
||||
db: Session,
|
||||
known_for: list | None,
|
||||
uid: int,
|
||||
movie_lib_ids: list[int],
|
||||
show_lib_ids: list[int],
|
||||
lang: str,
|
||||
) -> list[dict]:
|
||||
"""The person's TMDB "Known for" credits (top by popularity) that are IN our library, as cards, in
|
||||
popularity order. Matched by the canonical guid `tmdb:<id>`; out-of-library credits are dropped."""
|
||||
"""The person's TMDB "Known for" credits (most-voted first) that are IN our library, as cards, in
|
||||
that order, at most _KNOWN_FOR_SHOWN. Matched by the canonical guid `tmdb:<id>`; out-of-library
|
||||
credits are dropped."""
|
||||
if not known_for:
|
||||
return []
|
||||
# Match each credit to a library row by guid, bucketed by media_type (movie ids and tv ids collide
|
||||
@@ -50,10 +57,10 @@ def _resolve_known_for(
|
||||
# A credit resolves against its OWN media_type's bucket only.
|
||||
if k.get("media_type") == "tv":
|
||||
if g in shows:
|
||||
out.append(show_card(shows[g], show_statuses.get(shows[g].id, "new")))
|
||||
out.append(show_card(shows[g], show_statuses.get(shows[g].id, "new"), lang=lang))
|
||||
elif g in movies:
|
||||
out.append(movie_card(movies[g], m_states.get(movies[g].id)))
|
||||
return out
|
||||
out.append(movie_card(movies[g], m_states.get(movies[g].id), lang=lang))
|
||||
return out[:_KNOWN_FOR_SHOWN]
|
||||
|
||||
|
||||
@router.get("/people/cards")
|
||||
@@ -159,6 +166,7 @@ def _enrich_person_bg(person_id: int) -> None:
|
||||
def person_detail(
|
||||
name: str,
|
||||
background_tasks: BackgroundTasks,
|
||||
lang: str = "en",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -171,18 +179,24 @@ def person_detail(
|
||||
E5 S6: the first time a page is opened, the person's TMDB bio + "Known for" credits are lazily
|
||||
fetched (from the S2-linked `tmdb_person_id`) and persisted, then served from the row on every open.
|
||||
The fetch runs OFF the request path (a background task); the first open serves whatever's in the row
|
||||
(bio possibly still null) and the bio lands from the row by the next open.
|
||||
with `enriching` set, and the page asks again until the bio lands. E30 S6: the people_enrich job
|
||||
fetches everyone ahead of time, so this lazy path is only for a person the job hasn't reached yet.
|
||||
"""
|
||||
lang = norm_lang(lang)
|
||||
person = db.execute(select(PlexPerson).where(PlexPerson.name == name)).scalars().first()
|
||||
# Lazy one-time bio enrich, scheduled AFTER the response so the TMDB call is never on the serve path.
|
||||
# Gated by a per-person cooldown so concurrent/rapid opens (or repeated opens while TMDB is degraded)
|
||||
# can't pile background tasks onto the shared threadpool.
|
||||
if (
|
||||
# `enriching` tells the page the bio is on its way (E30 S6): it asks again shortly instead of showing
|
||||
# an empty page until the next open. True while a fetch is pending, including one an earlier open
|
||||
# already reserved.
|
||||
enriching = (
|
||||
person is not None
|
||||
and person.tmdb_person_id
|
||||
and bool(person.tmdb_person_id)
|
||||
and person.tmdb_enriched_at is None
|
||||
and _should_schedule_enrich(person.id)
|
||||
):
|
||||
and bool(sysconfig.get_str(db, "tmdb_api_key"))
|
||||
)
|
||||
if enriching and person is not None and _should_schedule_enrich(person.id):
|
||||
background_tasks.add_task(_enrich_person_bg, person.id)
|
||||
movie_lib_ids, show_lib_ids = content_libs(db)
|
||||
|
||||
@@ -199,14 +213,16 @@ def person_detail(
|
||||
PlexItem.directors.contains([name]),
|
||||
),
|
||||
)
|
||||
.order_by(PlexItem.year.desc().nullslast(), func.lower(PlexItem.title))
|
||||
.order_by(
|
||||
PlexItem.year.desc().nullslast(), collated(title_col(PlexItem, lang), lang)
|
||||
)
|
||||
.limit(_PERSON_SECTION_MAX)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
states = rollup.item_state_map(db, user.id, [it.id for it in items])
|
||||
movies = [movie_card(it, states.get(it.id)) for it in items]
|
||||
movies = [movie_card(it, states.get(it.id), lang=lang) for it in items]
|
||||
|
||||
shows: list[dict] = []
|
||||
if show_lib_ids:
|
||||
@@ -220,28 +236,34 @@ def person_detail(
|
||||
PlexShow.directors.contains([name]),
|
||||
),
|
||||
)
|
||||
.order_by(PlexShow.year.desc().nullslast(), func.lower(PlexShow.title))
|
||||
.order_by(
|
||||
PlexShow.year.desc().nullslast(), collated(title_col(PlexShow, lang), lang)
|
||||
)
|
||||
.limit(_PERSON_SECTION_MAX)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
statuses = rollup.state_map(db, user.id, [s.id for s in show_rows])
|
||||
shows = [show_card(sh, statuses.get(sh.id, "new")) for sh in show_rows]
|
||||
shows = [show_card(sh, statuses.get(sh.id, "new"), lang=lang) for sh in show_rows]
|
||||
|
||||
known_for = _resolve_known_for(
|
||||
db, person.known_for if person else None, user.id, movie_lib_ids, show_lib_ids
|
||||
db, person.known_for if person else None, user.id, movie_lib_ids, show_lib_ids, lang
|
||||
)
|
||||
return {
|
||||
"name": name,
|
||||
"photo": person.photo_thumb if person else None,
|
||||
"biography": person.biography if person else None,
|
||||
# The Hungarian biography for a Hungarian reader, the instance-language one when TMDB has none.
|
||||
"biography": ((person.biography_hu if lang == "hu" else None) or person.biography)
|
||||
if person
|
||||
else None,
|
||||
"birthday": person.birthday if person else None,
|
||||
"deathday": person.deathday if person else None,
|
||||
"place_of_birth": person.place_of_birth if person else None,
|
||||
"known_for_department": person.known_for_department if person else None,
|
||||
"also_known_as": (person.also_known_as if person else None) or [],
|
||||
"known_for": known_for,
|
||||
"enriching": enriching,
|
||||
"movies": movies,
|
||||
"shows": shows,
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Plex playback: image/subtitle proxies, HLS streaming, and watch-state writes."""
|
||||
|
||||
import hashlib
|
||||
import threading
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
@@ -214,24 +213,31 @@ def _resume_push_superseded(user_id: int, item_id: int, intent: datetime) -> boo
|
||||
|
||||
@router.get("/person-image")
|
||||
def person_image(u: str, request: Request) -> Response:
|
||||
"""Proxy a cast/crew photo from Plex's public metadata CDN (host-whitelisted; no token needed).
|
||||
Signed-session auth (no DB) so it, like /image, never holds a pool connection during the fetch."""
|
||||
"""Proxy a cast/crew photo from Plex's public metadata CDN (only a headshot-shaped URL on its host;
|
||||
no token needed), resized through Plex and cached (plex.people.cache_portrait; the people_enrich job
|
||||
fills the same cache ahead of time). Signed-session auth; like /image, the DB session is released
|
||||
before the fetch."""
|
||||
_require_session(request)
|
||||
if not u.startswith(plex_people.PERSON_IMG_HOST):
|
||||
if not plex_people.is_portrait_url(u):
|
||||
raise refused(400, "image_host")
|
||||
cache_key = "person_" + hashlib.sha1(u.encode()).hexdigest()
|
||||
hit = imgcache.read(cache_key)
|
||||
hit = plex_people.portrait_cached(u)
|
||||
if hit:
|
||||
return imgcache.response(*hit)
|
||||
try:
|
||||
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
||||
r = c.get(u)
|
||||
r.raise_for_status()
|
||||
except httpx.HTTPError:
|
||||
raise refused(502, "image_fetch_failed")
|
||||
ct = r.headers.get("content-type", "image/jpeg")
|
||||
imgcache.write(cache_key, r.content, ct)
|
||||
return imgcache.response(r.content, ct)
|
||||
with SessionLocal() as db:
|
||||
plex: PlexClient | None = PlexClient(db)
|
||||
# silent-ok: Plex off is a setting; without its resize the original comes straight from the CDN.
|
||||
except PlexNotConfigured:
|
||||
plex = None
|
||||
try:
|
||||
if plex is None:
|
||||
data, ct = plex_people.cache_portrait(None, u)
|
||||
else:
|
||||
with plex:
|
||||
data, ct = plex_people.cache_portrait(plex, u)
|
||||
except (PlexError, httpx.HTTPError):
|
||||
raise refused(502, "image_fetch_failed") from None
|
||||
return imgcache.response(data, ct)
|
||||
|
||||
|
||||
def _push_resume_guarded(
|
||||
|
||||
@@ -3,8 +3,10 @@
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user
|
||||
@@ -17,7 +19,7 @@ from app.models import (
|
||||
PlexState,
|
||||
User,
|
||||
)
|
||||
from app.plex.cards import leaf_card
|
||||
from app.plex.cards import display_title, leaf_card, norm_lang
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.routes._common import owned_or_404, not_found
|
||||
from app.refusal import refused
|
||||
@@ -135,8 +137,12 @@ def create_playlist(
|
||||
|
||||
@router.get("/playlists/{pid}")
|
||||
def playlist_detail(
|
||||
pid: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
pid: int,
|
||||
lang: str = "en",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
lang = norm_lang(lang)
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
rows = db.execute(_playlist_item_query(pid)).all()
|
||||
items = [it for (_pi, it) in rows]
|
||||
@@ -161,8 +167,9 @@ def playlist_detail(
|
||||
leaf_card(
|
||||
it,
|
||||
states.get(it.id),
|
||||
shows[it.show_id].title if it.show_id in shows else None,
|
||||
display_title(shows[it.show_id], lang) if it.show_id in shows else None,
|
||||
shows[it.show_id].rating_key if it.show_id in shows else None,
|
||||
lang=lang,
|
||||
)
|
||||
for it in items
|
||||
]
|
||||
@@ -217,7 +224,8 @@ def playlist_add_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
# Locked (like every add / reorder here): two adds otherwise read the same max position.
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
it = by_rating_key(db, PlexItem, body.item_rating_key)
|
||||
if it is None:
|
||||
raise not_found("item")
|
||||
@@ -249,7 +257,9 @@ def playlist_remove_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
# Locked like the adds and the reorder: the list row first, then its items — a remove that
|
||||
# locked an item row before a reorder locked the list deadlocked against it.
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
it = by_rating_key(db, PlexItem, item_rating_key)
|
||||
if it is not None:
|
||||
db.execute(
|
||||
@@ -285,16 +295,75 @@ def _items_by_rks(db: Session, rks: list) -> list[PlexItem]:
|
||||
return out
|
||||
|
||||
|
||||
class PlexItemAddBulkIn(PlexItemKeysIn):
|
||||
# Where each key goes back (E28 S3e2, the list page's "Undo" of a removal): an index over the
|
||||
# ORDERED rows, one per key. The two-call shape (append, then send the whole order) leaves the rows
|
||||
# at the end whenever the second call fails, and the order it sends would overwrite anything
|
||||
# another tab changed in between.
|
||||
positions: list[Annotated[int, Field(ge=0)]] | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _one_position_per_key(self):
|
||||
if self.positions is not None and len(self.positions) != len(self.item_rating_keys):
|
||||
raise ValueError("positions must pair one-to-one with item_rating_keys")
|
||||
return self
|
||||
|
||||
|
||||
def _insert_at(db: Session, pid: int, placed: list[tuple[int, PlexItem]]) -> int:
|
||||
"""Insert new rows at their indices (ascending, so each index is where the row stood when the
|
||||
rows before it were back), renumber the whole list 0..n-1, and say how many were added. An item
|
||||
already in the list does not move: the add stays idempotent."""
|
||||
rows = list(
|
||||
db.execute(
|
||||
select(PlexPlaylistItem)
|
||||
.where(PlexPlaylistItem.playlist_id == pid)
|
||||
.order_by(PlexPlaylistItem.position, PlexPlaylistItem.id)
|
||||
).scalars()
|
||||
)
|
||||
present = {r.item_id for r in rows}
|
||||
added = 0
|
||||
for at, it in sorted(placed, key=lambda p: p[0]):
|
||||
if it.id in present:
|
||||
continue
|
||||
row = PlexPlaylistItem(playlist_id=pid, item_id=it.id, position=0)
|
||||
db.add(row)
|
||||
rows.insert(min(at, len(rows)), row)
|
||||
present.add(it.id)
|
||||
added += 1
|
||||
for i, r in enumerate(rows):
|
||||
r.position = i
|
||||
return added
|
||||
|
||||
|
||||
@router.post("/playlists/{pid}/items/bulk")
|
||||
def playlist_add_bulk(
|
||||
pid: int,
|
||||
body: PlexItemKeysIn,
|
||||
body: PlexItemAddBulkIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Append many items at once (a whole season/show). Already-present items are skipped; the input
|
||||
order is preserved for the newly-added ones. Returns how many were added + the new total."""
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
order is preserved for the newly-added ones. With `positions`, each key is put back at its index
|
||||
instead of appended. Returns how many were added + the new total."""
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
if body.positions is not None:
|
||||
by_rk = {it.rating_key: it for it in _items_by_rks(db, body.item_rating_keys)}
|
||||
placed = [
|
||||
(at, by_rk[str(rk)])
|
||||
for rk, at in zip(body.item_rating_keys, body.positions, strict=True)
|
||||
if str(rk) in by_rk
|
||||
]
|
||||
added = _insert_at(db, pid, placed)
|
||||
if added:
|
||||
pl.updated_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
total = (
|
||||
db.scalar(
|
||||
select(func.count(PlexPlaylistItem.id)).where(PlexPlaylistItem.playlist_id == pid)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return {"ok": True, "added": added, "item_count": total}
|
||||
items = _items_by_rks(db, body.item_rating_keys)
|
||||
existing = set(
|
||||
db.execute(
|
||||
@@ -337,7 +406,7 @@ def playlist_remove_bulk(
|
||||
) -> dict:
|
||||
"""Remove many items at once (a whole season/show, or a multi-select). Returns how many were
|
||||
removed + the new total."""
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
ids = [it.id for it in _items_by_rks(db, body.item_rating_keys)]
|
||||
removed = 0
|
||||
if ids:
|
||||
@@ -367,7 +436,7 @@ def reorder_playlist(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Set the playlist order from an explicit list of item rating_keys (0-based positions)."""
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||
order = body.item_rating_keys
|
||||
id_by_rk = {
|
||||
it.rating_key: it.id
|
||||
@@ -375,14 +444,27 @@ def reorder_playlist(
|
||||
select(PlexItem).where(PlexItem.rating_key.in_(order or [""]))
|
||||
).scalars()
|
||||
}
|
||||
for pos, rk in enumerate(order):
|
||||
iid = id_by_rk.get(rk)
|
||||
if iid is not None:
|
||||
db.execute(
|
||||
update(PlexPlaylistItem)
|
||||
.where(PlexPlaylistItem.playlist_id == pid, PlexPlaylistItem.item_id == iid)
|
||||
.values(position=pos)
|
||||
)
|
||||
rows = {
|
||||
r.item_id: r
|
||||
for r in db.execute(
|
||||
select(PlexPlaylistItem).where(PlexPlaylistItem.playlist_id == pid)
|
||||
).scalars()
|
||||
}
|
||||
pos = 0
|
||||
seen: set[int] = set()
|
||||
for rk in order:
|
||||
row = rows.get(id_by_rk.get(rk, -1))
|
||||
if row is not None and row.item_id not in seen:
|
||||
row.position = pos
|
||||
pos += 1
|
||||
seen.add(row.item_id)
|
||||
# Rows the payload omitted (added in another tab meanwhile) keep their relative order after
|
||||
# the given ones, so no two rows share a position (the YouTube reorder does the same).
|
||||
for row in sorted(
|
||||
(r for iid, r in rows.items() if iid not in seen), key=lambda r: (r.position, r.id)
|
||||
):
|
||||
row.position = pos
|
||||
pos += 1
|
||||
pl.updated_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
@@ -130,7 +130,7 @@ def _ingest_candidates(
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
|
||||
details = yt.get_channels(new_channel_ids)
|
||||
apply_channel_details(db, details)
|
||||
apply_channel_details(db, details, new_channel_ids)
|
||||
db.commit()
|
||||
except YouTubeError:
|
||||
# The stubs remain and the details can be backfilled later — but until they are, those
|
||||
|
||||
@@ -71,7 +71,7 @@ def sync_status(user: User = Depends(current_user), db: Session = Depends(get_db
|
||||
def my_status(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Per-user sync progress: how far the channels *this* user subscribes to have synced."""
|
||||
sub_join = and_(Subscription.channel_id == Channel.id, Subscription.user_id == user.id)
|
||||
total, details_synced, recent_synced, deep_done = db.execute(
|
||||
total, details_synced, recent_synced, all_deep_done = db.execute(
|
||||
select(
|
||||
func.count(Channel.id),
|
||||
func.count(Channel.details_synced_at),
|
||||
@@ -82,7 +82,7 @@ def my_status(user: User = Depends(current_user), db: Session = Depends(get_db))
|
||||
.join(Subscription, sub_join)
|
||||
).one()
|
||||
total = total or 0
|
||||
deep_done = int(deep_done or 0)
|
||||
all_deep_done = int(all_deep_done or 0)
|
||||
|
||||
# Per-user full-history (deep) backfill: channels this user has opted in, and an ETA
|
||||
# for the still-pending ones (quota-bound estimate).
|
||||
@@ -97,6 +97,10 @@ def my_status(user: User = Depends(current_user), db: Session = Depends(get_db))
|
||||
.all()
|
||||
)
|
||||
deep_eta = estimate_deep_backfill(db, deep_requested_channels)
|
||||
# "done / requested" is shown as one ratio, so both halves count the SAME channels: the ones this
|
||||
# user asked full history for. Counting every subscribed channel whose back-catalog happens to be
|
||||
# complete (another user asked, or it was short) gave "342/329".
|
||||
deep_done = sum(1 for c in deep_requested_channels if c.backfill_done)
|
||||
|
||||
my_videos = db.scalar(
|
||||
select(func.count(Video.id)).join(
|
||||
@@ -117,7 +121,7 @@ def my_status(user: User = Depends(current_user), db: Session = Depends(get_db))
|
||||
"channels_recent_synced": recent_synced,
|
||||
"channels_deep_done": deep_done,
|
||||
"channels_recent_pending": total - recent_synced,
|
||||
"channels_deep_pending": total - deep_done,
|
||||
"channels_deep_pending": total - all_deep_done,
|
||||
"channels_deep_requested": len(deep_requested_channels),
|
||||
"deep_pending_count": deep_eta["channels_pending"],
|
||||
"deep_eta_seconds": deep_eta["eta_seconds"],
|
||||
|
||||
@@ -18,6 +18,7 @@ from app.downloads.gc import run_download_gc
|
||||
from app.models import SchedulerSetting, User
|
||||
from app.plex.sync import sync as run_plex_sync
|
||||
from app.plex.tmdb import run_enrich as run_tmdb_enrich
|
||||
from app.plex.tmdb import run_people_enrich
|
||||
from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync
|
||||
from app.notifications import create_notification
|
||||
from app.state import is_sync_paused
|
||||
@@ -69,6 +70,7 @@ JOB_INTERVALS: dict[str, int] = {
|
||||
"plex_watch_sync": settings.plex_watch_sync_interval_min,
|
||||
"plex_watch_reconcile": settings.plex_watch_reconcile_interval_min,
|
||||
"tmdb_enrich": settings.tmdb_enrich_interval_min,
|
||||
"people_enrich": settings.people_enrich_interval_min,
|
||||
"audit_gc": settings.audit_gc_minutes,
|
||||
"log_gc": settings.log_gc_minutes,
|
||||
}
|
||||
@@ -115,7 +117,9 @@ def _record(name: str, **fields) -> None:
|
||||
_activity.setdefault(name, {}).update(fields)
|
||||
|
||||
|
||||
def _notify_done(db, actor_id: int, job_id: str, status: str, summary: str | None) -> None:
|
||||
def _notify_done(
|
||||
db, actor_id: int, job_id: str, status: str, summary: str | None, result: dict | None = None
|
||||
) -> None:
|
||||
"""Post a completion notice to the triggering admin's inbox (manual runs only). Best
|
||||
effort — a notification failure must never mask the job's own outcome."""
|
||||
try:
|
||||
@@ -125,7 +129,7 @@ def _notify_done(db, actor_id: int, job_id: str, status: str, summary: str | Non
|
||||
type="scheduler",
|
||||
title=f"{job_id}: {status}", # English fallback; UI renders translated from data
|
||||
body=summary,
|
||||
data={"job_id": job_id, "status": status, "summary": summary},
|
||||
data={"job_id": job_id, "status": status, "summary": summary, "result": result},
|
||||
)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
@@ -140,7 +144,12 @@ def _run_changed(result) -> bool:
|
||||
if result is None:
|
||||
return False
|
||||
if isinstance(result, dict):
|
||||
return any(isinstance(v, (int, float)) and v for v in result.values())
|
||||
# A phased job (backfill's recent/deep, people_enrich's bios/portraits) nests a result per
|
||||
# phase: a count inside a phase is a change too.
|
||||
return any(
|
||||
_run_changed(v) if isinstance(v, dict) else isinstance(v, (int, float)) and bool(v)
|
||||
for v in result.values()
|
||||
)
|
||||
if isinstance(result, (int, float)):
|
||||
return result != 0
|
||||
return bool(result)
|
||||
@@ -152,6 +161,9 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
||||
they changed something, so routine no-op polls don't spam the trail. Best-effort."""
|
||||
if actor_id is None and status == "ok" and not _run_changed(result):
|
||||
return
|
||||
# The result as data too (E30): the audit log names it in the reader's language, as the inbox
|
||||
# does; `summary` stays the English line of the row's own summary column.
|
||||
data = _result_data(result)
|
||||
try:
|
||||
audit.record(
|
||||
db,
|
||||
@@ -160,7 +172,7 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
||||
target_type="job",
|
||||
target_id=job_id,
|
||||
summary=f"{job_id}: {status}" + (f" — {summary}" if summary else ""),
|
||||
after={"status": status, "summary": summary},
|
||||
after={"status": status, "summary": summary, **({"result": data} if data else {})},
|
||||
)
|
||||
db.commit()
|
||||
except Exception:
|
||||
@@ -192,25 +204,41 @@ def _job(name: str, fn) -> None:
|
||||
status="skipped",
|
||||
last_finished=_now_iso(),
|
||||
last_result="paused",
|
||||
result=_PAUSED,
|
||||
progress=None,
|
||||
)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "skipped", "sync paused")
|
||||
_audit_run(db, actor_id, name, "skipped", "sync paused", None)
|
||||
_notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
||||
_audit_run(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
||||
return
|
||||
with progress.bind(sink):
|
||||
result = fn(db)
|
||||
summary = _summarize(result)
|
||||
data = _result_data(result)
|
||||
log.info("job %s -> %s", name, result)
|
||||
_record(name, status="ok", last_finished=_now_iso(), last_result=summary, progress=None)
|
||||
_record(
|
||||
name,
|
||||
status="ok",
|
||||
last_finished=_now_iso(),
|
||||
last_result=summary,
|
||||
result=data,
|
||||
progress=None,
|
||||
)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "ok", summary)
|
||||
_notify_done(db, actor_id, name, "ok", summary, data)
|
||||
_audit_run(db, actor_id, name, "ok", summary, result)
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
|
||||
err = str(exc) or exc.__class__.__name__
|
||||
_record(name, status="error", last_finished=_now_iso(), last_error=err, progress=None)
|
||||
_record(
|
||||
name,
|
||||
status="error",
|
||||
last_finished=_now_iso(),
|
||||
last_error=err,
|
||||
result=None,
|
||||
progress=None,
|
||||
)
|
||||
if actor_id is not None:
|
||||
_notify_done(db, actor_id, name, "error", err)
|
||||
_audit_run(db, actor_id, name, "error", err, None)
|
||||
@@ -221,6 +249,21 @@ def _job(name: str, fn) -> None:
|
||||
db.close()
|
||||
|
||||
|
||||
# A skipped run's result, in the same shape as a job's own `{"skipped": …}`.
|
||||
_PAUSED = {"skipped": "sync paused"}
|
||||
|
||||
|
||||
def _result_data(result) -> dict | None:
|
||||
"""A job's result as the client names it in the reader's language (E30): the dict itself (counts,
|
||||
a `skipped` / `reason` word, a nested dict per phase), `enrich`'s bare count as one. The one-line
|
||||
`_summarize` is the English fallback."""
|
||||
if isinstance(result, dict):
|
||||
return result or None
|
||||
if isinstance(result, int) and not isinstance(result, bool):
|
||||
return {"enriched": result}
|
||||
return None
|
||||
|
||||
|
||||
def _summarize(result) -> str | None:
|
||||
"""Compact one-line rendering of a job's return value for the dashboard."""
|
||||
if result is None:
|
||||
@@ -259,6 +302,7 @@ def scheduler_snapshot() -> dict:
|
||||
"last_started": a.get("last_started"),
|
||||
"last_finished": a.get("last_finished"),
|
||||
"last_result": a.get("last_result"),
|
||||
"result": a.get("result"),
|
||||
"last_error": a.get("last_error"),
|
||||
"progress": a.get("progress"),
|
||||
}
|
||||
@@ -369,6 +413,12 @@ def _tmdb_enrich_job() -> None:
|
||||
_job("tmdb_enrich", run_tmdb_enrich)
|
||||
|
||||
|
||||
def _people_enrich_job() -> None:
|
||||
# Make the people local (E30 S6): a batch of person bios from TMDB and a batch of portraits resized
|
||||
# through Plex, so a person page and a cast strip never wait on an outside service.
|
||||
_job("people_enrich", run_people_enrich)
|
||||
|
||||
|
||||
def _audit_gc_job() -> None:
|
||||
# Prune audit-log rows older than the admin-set retention (audit_retention_days; 0 = keep
|
||||
# forever). Pure DB, no quota.
|
||||
@@ -415,6 +465,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = {
|
||||
"plex_watch_sync": _plex_watch_sync_job,
|
||||
"plex_watch_reconcile": _plex_watch_reconcile_job,
|
||||
"tmdb_enrich": _tmdb_enrich_job,
|
||||
"people_enrich": _people_enrich_job,
|
||||
"audit_gc": _audit_gc_job,
|
||||
"log_gc": _log_gc_job,
|
||||
}
|
||||
|
||||
@@ -115,7 +115,9 @@ _TOPIC_SLUGS = {
|
||||
"Tourism": "Travel",
|
||||
"Fashion": "Fashion & Beauty",
|
||||
"Physical_attractiveness": "Fashion & Beauty",
|
||||
"Humor": "Comedy",
|
||||
# YouTube links the British page (…/wiki/Humour); the key was "Humor", so no channel ever got Comedy
|
||||
# from its topic (E30).
|
||||
"Humour": "Comedy",
|
||||
"Entertainment": "Entertainment",
|
||||
"Film": "Film",
|
||||
"Television_program": "TV",
|
||||
|
||||
+185
-31
@@ -8,13 +8,15 @@ ingested (with a stub channel) so the mirror is faithful."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import TypeVar
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import Engine, create_engine, delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.pool import NullPool
|
||||
|
||||
from app import logs, progress, quota
|
||||
from app.auth import has_read_scope
|
||||
@@ -27,6 +29,87 @@ log = logs.get_logger(__name__)
|
||||
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
||||
WRITE_UNIT_COST = 50
|
||||
|
||||
# The advisory-lock namespaces ("PL" per playlist, "PU" per user), so a key cannot meet another
|
||||
# feature's lock on the same number (Postgres keeps the two-int keys apart from the one-bigint ones).
|
||||
_SYNC_LOCK_NS = 0x504C
|
||||
_USER_SYNC_LOCK_NS = 0x5055
|
||||
|
||||
# The claims' own connections, outside the app pool: a claim is held for a whole push (seconds of
|
||||
# YouTube calls) beside the request's pooled session and quota's, so a burst of pushes drawing a
|
||||
# third pooled connection each ran the pool dry and stalled every other request for pool_timeout.
|
||||
# NullPool opens one per claim and closes it after, so a lock can never ride back into a pool either.
|
||||
_claim_engines: dict[str, Engine] = {}
|
||||
|
||||
|
||||
def _claim_engine(db: Session) -> Engine:
|
||||
url = db.get_bind().engine.url
|
||||
key = url.render_as_string(hide_password=False)
|
||||
eng = _claim_engines.get(key)
|
||||
if eng is None:
|
||||
eng = _claim_engines[key] = create_engine(url, poolclass=NullPool, hide_parameters=True)
|
||||
return eng
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _advisory_claim(db: Session, ns: int, key: int) -> Iterator[bool]:
|
||||
"""Try a session-level advisory lock for the block on a connection of its own; yields whether it
|
||||
was free. It survives the block's commits, and a crashed worker's lock dies with its connection
|
||||
— no claim column that a crash leaves stuck."""
|
||||
conn = _claim_engine(db).connect()
|
||||
held = False
|
||||
try:
|
||||
held = bool(conn.scalar(select(func.pg_try_advisory_lock(ns, key))))
|
||||
conn.commit()
|
||||
yield held
|
||||
finally:
|
||||
try:
|
||||
if held:
|
||||
conn.scalar(select(func.pg_advisory_unlock(ns, key)))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def sync_claim(db: Session, playlist_id: int) -> AbstractContextManager[bool]:
|
||||
"""Claim one playlist's YouTube sync (push, re-pull, the scheduled mirror refresh) for the block;
|
||||
yields whether it was free. A push reads YouTube, then writes both sides for seconds and commits
|
||||
step by step, so a row lock ends at its first commit and two tabs (or a push and the scheduler)
|
||||
interleave into a doubled YouTube item or a baseline recorded over the other's writes."""
|
||||
return _advisory_claim(db, _SYNC_LOCK_NS, playlist_id)
|
||||
|
||||
|
||||
class PlaylistGone(Exception):
|
||||
"""The list a sync was working on was deleted meanwhile (another tab, the mirror drop). Its
|
||||
items went with it, so a plan made now reads "every item removed here" — and would delete them
|
||||
all from the YouTube playlist the reader chose to keep. The route answers 404."""
|
||||
|
||||
|
||||
def lock_playlists(db: Session, *playlist_ids: int) -> set[int]:
|
||||
"""Row-lock playlists before touching their items, several in id order. Every item write takes
|
||||
the PLAYLIST first — the route edits, a push, a re-pull, the scheduled mirror refresh and a
|
||||
delete alike: a writer that locked an item row first and the playlist second (a push's item
|
||||
replace beside a remove) deadlocks against one that went the other way, and the loser is a
|
||||
500. The rows are re-read, so `dirty`, the name and the sync baseline are the ones a finished
|
||||
writer committed. Held to the caller's next commit — so no commit may come between this and
|
||||
the write it guards. Returns the ids it found: a list deleted meanwhile is not among them."""
|
||||
rows = db.scalars(
|
||||
select(Playlist)
|
||||
.where(Playlist.id.in_(sorted(set(playlist_ids))))
|
||||
.order_by(Playlist.id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).all() # consumed: rows fetched are what refreshes the loaded objects
|
||||
return {pl.id for pl in rows}
|
||||
|
||||
|
||||
def _lock_or_gone(db: Session, pl: Playlist) -> None:
|
||||
"""`lock_playlists` for a sync that goes on to write YouTube or the items: a list deleted since
|
||||
it was read raises `PlaylistGone` (rolled back) instead of being planned as an empty one."""
|
||||
pl_id = pl.id # read first: the rollback expires `pl`, and its row is gone
|
||||
if pl_id not in lock_playlists(db, pl_id):
|
||||
db.rollback()
|
||||
raise PlaylistGone(pl_id)
|
||||
|
||||
|
||||
def fingerprint(name: str, ids: list[str]) -> str:
|
||||
"""A stable checksum of a playlist's name + ordered video ids. Stored as the synced
|
||||
@@ -91,11 +174,18 @@ def parse_unavailable(pl: Playlist) -> set[str]:
|
||||
return _load_baseline(pl)[1]
|
||||
|
||||
|
||||
def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None = None) -> None:
|
||||
def _set_baseline(
|
||||
pl: Playlist,
|
||||
ids: list[str],
|
||||
unavailable: list[str] | None = None,
|
||||
*,
|
||||
name: str | None = None,
|
||||
) -> None:
|
||||
"""Record `ids` as both the hash (drives `dirty`, over the local ids only) and the raw ordered
|
||||
set (drives merge). `unavailable` = un-ingestable-but-live YouTube ids to carry forward so the
|
||||
next merge skips them; stored only when non-empty (keeps the common case a plain list)."""
|
||||
pl.synced_fingerprint = fingerprint(pl.name, ids)
|
||||
next merge skips them; stored only when non-empty (keeps the common case a plain list). `name`:
|
||||
the title YouTube now has, when that may differ from the list's own (see the push)."""
|
||||
pl.synced_fingerprint = fingerprint(pl.name if name is None else name, ids)
|
||||
if unavailable:
|
||||
pl.synced_video_ids = json.dumps({"ids": ids, "unavailable": list(unavailable)})
|
||||
else:
|
||||
@@ -103,6 +193,25 @@ def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None =
|
||||
pl.dirty = False
|
||||
|
||||
|
||||
def _set_baseline_under_lock(
|
||||
db: Session,
|
||||
pl: Playlist,
|
||||
ids: list[str],
|
||||
unavailable: list[str] | None = None,
|
||||
*,
|
||||
name: str,
|
||||
) -> None:
|
||||
"""A push's closing baseline: `ids` and `name` are what YouTube now holds. The push spent
|
||||
seconds on YouTube writes since it replaced the local items, so an edit may have landed
|
||||
meanwhile — lock the list and derive `dirty` from what is there, or the edit would read as
|
||||
synced and never reach YouTube. The name too: the lock re-reads a rename made meanwhile, and a
|
||||
baseline taken over THAT name matched it, so the list read clean with YouTube's title still the
|
||||
old one (E28 review round 4)."""
|
||||
_lock_or_gone(db, pl)
|
||||
_set_baseline(pl, ids, unavailable, name=name)
|
||||
recompute_dirty(db, pl)
|
||||
|
||||
|
||||
def recompute_dirty(db: Session, pl: Playlist) -> None:
|
||||
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
|
||||
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
|
||||
@@ -159,15 +268,13 @@ def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None
|
||||
db.commit()
|
||||
|
||||
|
||||
def _replace_items_from_youtube(
|
||||
db: Session, yt: YouTubeClient, pl: Playlist, ids: list[str]
|
||||
) -> list[str]:
|
||||
def _replace_items_from_youtube(db: Session, pl: Playlist, ids: list[str]) -> list[str]:
|
||||
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
|
||||
replace): ensure the videos exist in the catalog, keep only those present in their YT
|
||||
order (de-duplicated), swap the items out, and reset the synced fingerprint + dirty flag.
|
||||
`pl.name` must already be set to the desired name. The caller commits. Returns the final
|
||||
ordered ids."""
|
||||
_ensure_videos(db, yt, ids)
|
||||
replace): keep only those present in the catalog in their YT order (de-duplicated), swap the
|
||||
items out, and reset the synced fingerprint + dirty flag. The caller has ingested `ids`
|
||||
(`_ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
|
||||
after: an ingest in here re-fetched every id YouTube will not return (a private or deleted
|
||||
video) and its commit let go of the lock before the replace. Returns the final ordered ids."""
|
||||
present = (
|
||||
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
|
||||
)
|
||||
@@ -196,6 +303,21 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
||||
the user's local playlists and the built-in Watch later untouched."""
|
||||
if not has_read_scope(user):
|
||||
return {"synced": 0, "reason": "no read scope"}
|
||||
with user_sync_claim(db, user.id) as free:
|
||||
if not free:
|
||||
return {"synced": 0, "reason": "busy"}
|
||||
return pull_user_playlists(db, user)
|
||||
|
||||
|
||||
def user_sync_claim(db: Session, user_id: int) -> AbstractContextManager[bool]:
|
||||
"""Claim a user's bulk mirror pull for the block; yields whether it was free. Two runs at once
|
||||
(the scheduler beside a manual "sync now", two tabs) both saw a YouTube list unmirrored and
|
||||
each created its mirror — a duplicate row."""
|
||||
return _advisory_claim(db, _USER_SYNC_LOCK_NS, user_id)
|
||||
|
||||
|
||||
def pull_user_playlists(db: Session, user: User) -> dict:
|
||||
"""The mirror pull itself; the caller holds `user_sync_claim`."""
|
||||
synced = 0
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt_playlists = list(yt.iter_my_playlists())
|
||||
@@ -219,11 +341,15 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
# Drop mirrors whose YouTube playlist no longer exists.
|
||||
for ytid, pl in existing.items():
|
||||
if ytid not in yt_ids:
|
||||
db.delete(pl)
|
||||
db.flush()
|
||||
# Drop mirrors whose YouTube playlist no longer exists — locked first, as every item write.
|
||||
gone = [pl for ytid, pl in existing.items() if ytid not in yt_ids]
|
||||
if gone:
|
||||
lock_playlists(db, *(pl.id for pl in gone))
|
||||
for pl in gone:
|
||||
db.delete(pl)
|
||||
# Committed now: a run where no list below commits (all dirty, or all claimed) would
|
||||
# otherwise drop the deletions with the session.
|
||||
db.commit()
|
||||
|
||||
for idx, p in enumerate(yt_playlists):
|
||||
ytid = p.get("id")
|
||||
@@ -245,12 +371,27 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
||||
)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
else:
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
# The user is pushing or re-pulling this very list: it writes the same rows.
|
||||
# The next scheduled run refreshes it.
|
||||
continue
|
||||
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
||||
# Ingest first — it commits — so the row lock below is held from the dirty check
|
||||
# to the commit that replaces the items.
|
||||
_ensure_videos(db, yt, ids)
|
||||
# `dirty` read above is a snapshot from before the YouTube calls (sessions keep
|
||||
# loaded rows past a commit): a remove or reorder made meanwhile must not be
|
||||
# replaced by YouTube's order. Item writes lock the playlist row, so this waits for
|
||||
# one in flight and sees what it committed.
|
||||
lock_playlists(db, pl.id)
|
||||
if pl.dirty:
|
||||
db.rollback()
|
||||
continue
|
||||
pl.name = p.get("title") or pl.name
|
||||
# Mirror is authoritative: replace the items to match YouTube's order.
|
||||
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
||||
_replace_items_from_youtube(db, yt, pl, ids)
|
||||
db.commit()
|
||||
# Mirror is authoritative: replace the items to match YouTube's order.
|
||||
_replace_items_from_youtube(db, pl, ids)
|
||||
db.commit()
|
||||
synced += 1
|
||||
return {"synced": synced}
|
||||
|
||||
@@ -488,7 +629,8 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
breaker = _Breaker(playlist_id=pl.id)
|
||||
with YouTubeClient(db, user) as yt:
|
||||
if not pl.yt_playlist_id:
|
||||
pl.yt_playlist_id = yt.create_playlist(pl.name)
|
||||
pushed_name = pl.name
|
||||
pl.yt_playlist_id = yt.create_playlist(pushed_name)
|
||||
db.commit()
|
||||
created = 1
|
||||
# Brand-new playlist: just append every local video in order.
|
||||
@@ -501,7 +643,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
if not isinstance(added, _Failed):
|
||||
inserted += 1
|
||||
if not failures:
|
||||
_set_baseline(pl, local)
|
||||
_set_baseline_under_lock(db, pl, local, name=pushed_name)
|
||||
db.commit()
|
||||
return {
|
||||
"created": created,
|
||||
@@ -542,6 +684,14 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
||||
# YouTube write trips the breaker.
|
||||
_ensure_videos(db, yt, mp.add_to_local)
|
||||
# `local` was read before the YouTube calls above: an add or remove committed meanwhile
|
||||
# (another tab) would be undone by the replace below. Lock the list — the ingest's commit is
|
||||
# behind us — and plan again from what is there now. Nothing new needs ingesting: a video
|
||||
# the re-read adds is already a local row. A list deleted meanwhile stops here, before a plan
|
||||
# over its now-empty items removes them all from YouTube.
|
||||
_lock_or_gone(db, pl)
|
||||
local = _local_video_ids(db, pl)
|
||||
mp = merge_plan(parse_baseline(pl), local, yt_ids, effective_unavailable)
|
||||
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
|
||||
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
|
||||
present = (
|
||||
@@ -555,10 +705,12 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
_set_local_items(db, pl, final_local)
|
||||
db.commit()
|
||||
|
||||
# --- YouTube side (metered, breaker-guarded).
|
||||
if snippet is not None and (snippet.get("title") or "") != pl.name:
|
||||
# --- YouTube side (metered, breaker-guarded). The name as of the lock above: the closing
|
||||
# baseline records THIS one, whatever a rename made during the writes below says.
|
||||
pushed_name = pl.name
|
||||
if snippet is not None and (snippet.get("title") or "") != pushed_name:
|
||||
try:
|
||||
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
|
||||
yt.update_playlist_title(pl.yt_playlist_id, pushed_name)
|
||||
except YouTubeError as exc:
|
||||
# All the caller passes on is the word "title", which says the rename did not
|
||||
# happen but not that YouTube refused it, or why.
|
||||
@@ -626,7 +778,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||
(effective_unavailable | {v for v in mp.add_to_local if v not in present})
|
||||
& yt_id_set
|
||||
)
|
||||
_set_baseline(pl, final_local, unavailable)
|
||||
_set_baseline_under_lock(db, pl, final_local, unavailable, name=pushed_name)
|
||||
db.commit()
|
||||
return {
|
||||
"created": created,
|
||||
@@ -652,9 +804,11 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||
if snippet is None:
|
||||
raise YouTubeError("Playlist no longer exists on YouTube")
|
||||
pl.name = snippet.get("title") or pl.name
|
||||
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
||||
ordered = _replace_items_from_youtube(db, yt, pl, ids)
|
||||
_ensure_videos(db, yt, ids)
|
||||
_lock_or_gone(db, pl)
|
||||
pl.name = snippet.get("title") or pl.name
|
||||
ordered = _replace_items_from_youtube(db, pl, ids)
|
||||
db.commit()
|
||||
return {"items": len(ordered)}
|
||||
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
@@ -47,8 +48,37 @@ def _external_links(branding: dict) -> list | None:
|
||||
return out or None
|
||||
|
||||
|
||||
def apply_channel_details(db: Session, items: list[dict]) -> None:
|
||||
# How old a channel's metadata may get before it is fetched again. Banners, avatars, descriptions and
|
||||
# counts change on YouTube, and an old banner URL stops resolving at all — without a refresh a channel
|
||||
# kept whatever it had the day it was first seen. A week keeps counts roughly current at almost no
|
||||
# cost: `channels.list` is 1 quota unit per 50 channels, and only the stale ones are asked for.
|
||||
DETAILS_MAX_AGE = timedelta(days=7)
|
||||
|
||||
|
||||
def details_stale(channel: Channel, now: datetime) -> bool:
|
||||
"""Whether `channel`'s metadata is missing or older than DETAILS_MAX_AGE (see `details_stale_clause`)."""
|
||||
synced = channel.details_synced_at
|
||||
return synced is None or synced < now - DETAILS_MAX_AGE
|
||||
|
||||
|
||||
def details_stale_clause(now: datetime):
|
||||
"""`details_stale` as a WHERE clause, so the batch import and a single channel agree on "stale"."""
|
||||
return or_(
|
||||
Channel.details_synced_at.is_(None),
|
||||
Channel.details_synced_at < now - DETAILS_MAX_AGE,
|
||||
)
|
||||
|
||||
|
||||
def apply_channel_details(db: Session, items: list[dict], asked: Iterable[str]) -> None:
|
||||
"""Write `channels.list`'s answer onto the rows. `asked` is every id the call asked for: one
|
||||
YouTube did not return (terminated, deleted) is stamped as synced too — otherwise it stays
|
||||
stale forever and every view / import asks for it again, spending quota on a known miss."""
|
||||
now = datetime.now(timezone.utc)
|
||||
returned = {item["id"] for item in items}
|
||||
for cid in set(asked) - returned:
|
||||
missing = db.get(Channel, cid)
|
||||
if missing is not None:
|
||||
missing.details_synced_at = now
|
||||
for item in items:
|
||||
channel = db.get(Channel, item["id"])
|
||||
if channel is None:
|
||||
@@ -92,7 +122,7 @@ def should_prune(fetched_count: int, existing_count: int) -> bool:
|
||||
def import_subscriptions(db: Session, user: User) -> dict:
|
||||
"""Pull all subscriptions for `user`, upsert channels + subscription links, prune
|
||||
channels the user has unsubscribed from on YouTube, and fetch channel details for
|
||||
any channel we don't have details for yet."""
|
||||
any channel whose details are missing or stale (`DETAILS_MAX_AGE`)."""
|
||||
fetched: dict[str, dict] = {}
|
||||
new_channels = 0
|
||||
|
||||
@@ -145,12 +175,12 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
||||
removed += 1
|
||||
db.commit()
|
||||
|
||||
# Fetch details for channels that don't have them yet.
|
||||
# Fetch details for channels that don't have them yet, or had them too long ago.
|
||||
need_details = (
|
||||
db.execute(
|
||||
select(Channel).where(
|
||||
Channel.id.in_(fetched.keys()),
|
||||
Channel.details_synced_at.is_(None),
|
||||
details_stale_clause(datetime.now(timezone.utc)),
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
@@ -158,8 +188,9 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
||||
)
|
||||
detailed = 0
|
||||
if need_details:
|
||||
items = yt.get_channels([c.id for c in need_details])
|
||||
apply_channel_details(db, items)
|
||||
ids = [c.id for c in need_details]
|
||||
items = yt.get_channels(ids)
|
||||
apply_channel_details(db, items, ids)
|
||||
detailed = len(items)
|
||||
db.commit()
|
||||
|
||||
|
||||
+54
-10
@@ -137,10 +137,29 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
min=1,
|
||||
max=100_000,
|
||||
),
|
||||
# min=1, NOT 0: the per-file cap is enforced as `if cap and size > cap`, so a stored 0 would read
|
||||
# as falsy and SILENTLY disable the limit (an unbounded streamed write to the volume). Floor it at
|
||||
# 1 so the cap is always active; the overall footprint is separately bounded by the byte quota.
|
||||
ConfigSpec("download_upload_max_bytes", "int", "downloads", "download_upload_max_bytes", min=1),
|
||||
# min=1, NOT 0: a stored 0 used to read as "no per-file cap" (the cap was enforced as
|
||||
# `if cap and size > cap`, so a falsy value disabled it). Since P1 the route passes this number
|
||||
# straight to `capped_stream_to`, where 0 would refuse every upload instead — the opposite
|
||||
# failure, and the reason `config.download_upload_max_bytes` carries its own `ge=1`. Either way
|
||||
# a stored 0 is wrong, so it is floored at 1; the overall footprint is separately bounded by the
|
||||
# byte quota.
|
||||
# `max` is not cosmetic: `bodylimit.ROUTE_MAX_BODY_BYTES` refuses an upload body over exactly
|
||||
# this number at the edge, before the route's own check ever sees it. An admin allowed to raise
|
||||
# the knob past the edge would get 413s from a layer the admin page never mentions, so the two
|
||||
# are pinned together here and `tests/test_body_limit.py` holds them that way. Below the edge is
|
||||
# the useful direction and the one the knob exists for: a lowered knob is what makes the route's
|
||||
# own 422 — the one carrying this figure — the answer the user gets. `_coerce` also
|
||||
# REJECTS a stored value outside these bounds on read (falling back to the env default, with a
|
||||
# warning) — neither bound has always been what it is now, so a legal-when-written override can
|
||||
# be sitting in the database already.
|
||||
ConfigSpec(
|
||||
"download_upload_max_bytes",
|
||||
"int",
|
||||
"downloads",
|
||||
"download_upload_max_bytes",
|
||||
min=1,
|
||||
max=settings.download_upload_max_bytes,
|
||||
),
|
||||
ConfigSpec(
|
||||
"download_retention_days", "int", "downloads", "download_retention_days", min=0, max=3_650
|
||||
),
|
||||
@@ -236,9 +255,29 @@ def _choice(s: ConfigSpec, raw: object) -> str:
|
||||
return option
|
||||
|
||||
|
||||
class OutOfRange(ValueError):
|
||||
"""A stored int override that no longer satisfies its spec's bounds."""
|
||||
|
||||
|
||||
def _coerce(s: ConfigSpec, raw: str):
|
||||
if s.type == "int":
|
||||
return int(raw)
|
||||
# Bounds are checked on READ, not only in `set_value`. Bounds get ADDED to a spec
|
||||
# (download_upload_max_bytes gained a `max` when SEC-20 started enforcing the same number at
|
||||
# the edge; its `min` went 0 → 1 during E7) and env defaults get LOWERED, and either leaves
|
||||
# a stored value that was legal when it was written. Serving it is how one layer ends up
|
||||
# acting on a number another layer refuses, with the admin page showing a value outside the
|
||||
# range it renders.
|
||||
#
|
||||
# It raises rather than CLAMPING, and that distinction cost a round: clamping is a silent
|
||||
# rewrite, and silently rewriting is exactly what this module refuses to do elsewhere. The
|
||||
# case that settles it is a stored 0 for download_upload_max_bytes — legal when `min` was 0,
|
||||
# and meaning "no per-file cap" to the route that reads it. Clamped to `min=1` it becomes a
|
||||
# ONE-BYTE cap and uploads stop working with nothing in the log. Raising drops it into the
|
||||
# same fallback the uncoercible path uses: the env default, and one warning.
|
||||
v = int(raw)
|
||||
if (s.min is not None and v < s.min) or (s.max is not None and v > s.max):
|
||||
raise OutOfRange(f"{v} is outside [{s.min}, {s.max}]")
|
||||
return v
|
||||
if s.type == "bool":
|
||||
return raw == "true"
|
||||
if s.type == "choice":
|
||||
@@ -291,14 +330,19 @@ def get(db: Session, key: str):
|
||||
return _coerce(s, raw)
|
||||
except (TypeError, ValueError) as exc:
|
||||
# The admin set this in the UI and it is quietly not in force: the app runs on the env
|
||||
# default while the settings page goes on showing the value that was saved.
|
||||
# default while the settings page goes on showing the value that was saved. An OutOfRange is
|
||||
# the same situation arrived at from the other direction — the value was fine when it was
|
||||
# written and the spec moved under it — so it gets the same treatment and its own code.
|
||||
event = (
|
||||
"sysconfig.override_out_of_range"
|
||||
if isinstance(exc, OutOfRange)
|
||||
else "sysconfig.override_uncoercible"
|
||||
)
|
||||
if (s.key, raw) not in _coerce_warned:
|
||||
_coerce_warned.add((s.key, raw))
|
||||
log.warning(
|
||||
"a stored setting could not be coerced; the default is used instead",
|
||||
extra=logs.event(
|
||||
"sysconfig.override_uncoercible", key=s.key, reason=type(exc).__name__
|
||||
),
|
||||
"a stored setting is not usable; the default is used instead",
|
||||
extra=logs.event(event, key=s.key, reason=type(exc).__name__),
|
||||
)
|
||||
return _default(s)
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ from app.downloads.jobdb import (
|
||||
_staging_dir,
|
||||
)
|
||||
from app.downloads.status import STOPPED_BY_ROUTE, JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_stop = threading.Event()
|
||||
@@ -562,7 +562,7 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
|
||||
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
||||
if not job.display_name and asset.title:
|
||||
# display_name is VARCHAR(255); a title can be longer (e.g. a recipe in the title).
|
||||
fields["display_name"] = asset.title[:255]
|
||||
fields["display_name"] = asset.title[:DISPLAY_TEXT_MAX]
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
# Atomic finalize: only marks the job done while it is still running, so a concurrent
|
||||
# cancel/pause is never resurrected (see service.finalize_if_running).
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
// The backend type-check lane (E27 S3) — the gate's ninth lane, and the first thing that has ever
|
||||
// type-checked the ~36k lines of `app/`. Read as JSONC: pyright accepts comments here.
|
||||
//
|
||||
// basedpyright, not mypy: measured on this tree with the dependencies actually installed, both land
|
||||
// ~190-200 errors with the same distribution, so the choice was made on the things that differ —
|
||||
// basedpyright is ~2x faster, needs no MYPYPATH to resolve our `app.*` imports, and has a FIRST-CLASS
|
||||
// baseline (`--writebaseline`), which is the whole ratchet. mypy would have needed a third-party tool
|
||||
// for that.
|
||||
//
|
||||
// It runs in the pytest test image, never on the host: `requirements.lock` cannot be installed on
|
||||
// Windows at all (uvloop refuses to build there), so a host run would type-check against absent
|
||||
// third-party packages and report nonsense. The version is pinned in requirements-dev.txt, which the
|
||||
// image installs — so the lane's version IS the pin, with no host-vs-pin check to drift.
|
||||
//
|
||||
// `pythonVersion` is not free-standing: `/.python-version` is the single source of truth and
|
||||
// `tests/test_python_engine_contract.py` fails this gate if this file, the two Dockerfiles, the CI
|
||||
// container or ruff's target-version disagree with it.
|
||||
//
|
||||
// `include` covers `alembic` for the same reason the ruff lane was widened to it: migrations run on
|
||||
// every container start, so one that raises there crash-loops prod, and "it is generated code" is
|
||||
// not an argument when the generated code is what boots the system. `tests/` is deliberately OUT —
|
||||
// a test's job is to fail loudly when it is wrong, which is the opposite of the silent runtime
|
||||
// failure this lane exists to catch, and annotating fixtures to a checker's satisfaction is churn
|
||||
// the ratchet exists to avoid.
|
||||
{
|
||||
"include": ["app", "alembic"],
|
||||
"extraPaths": ["."],
|
||||
"typeCheckingMode": "basic",
|
||||
"pythonVersion": "3.12",
|
||||
"pythonPlatform": "Linux"
|
||||
}
|
||||
@@ -4,6 +4,10 @@
|
||||
# backend lint lane runs on the host (the gate fails when the host's ruff is not this one). httpx2:
|
||||
# Starlette 1.x's TestClient is built on it (`import httpx2 as httpx`) and only falls back to the
|
||||
# app's own httpx with a deprecation warning on every run.
|
||||
# basedpyright: the backend type-check lane (E27 S3). Pinned like everything else here — a type
|
||||
# checker whose version floats silently changes what "clean" means, and the ratchet baseline it
|
||||
# writes is keyed to the diagnostics THIS version produces.
|
||||
basedpyright==1.40.1
|
||||
pytest==8.4.2
|
||||
ruff==0.15.21
|
||||
httpx2==2.13.0
|
||||
|
||||
@@ -47,7 +47,6 @@ pydantic-settings==2.15.0
|
||||
pydantic==2.13.4
|
||||
pydantic_core==2.46.4
|
||||
python-dotenv==1.2.3
|
||||
python-multipart==0.0.32
|
||||
requests==2.34.2
|
||||
rich==15.0.0
|
||||
starlette==1.6.0
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
fastapi>=0.115,<1.0
|
||||
# Multipart form parsing — required by Starlette/FastAPI to accept file uploads (UploadFile/Form).
|
||||
# The download center's local-file Upload (E7) is the only multipart endpoint in the app.
|
||||
python-multipart>=0.0.9,<1.0
|
||||
uvicorn[standard]>=0.30,<1.0
|
||||
sqlalchemy>=2.0,<2.1
|
||||
psycopg[binary]>=3.2,<4.0
|
||||
|
||||
@@ -7,6 +7,10 @@
|
||||
# the repo wrap at the same width, and it's the width `ruff format` enforces (gated via
|
||||
# `ruff format --check` alongside `ruff check`).
|
||||
line-length = 100
|
||||
# The interpreter this code actually runs on, so ruff judges syntax and upgrade hints against the
|
||||
# right one. Left unset it falls back to a default that has nothing to do with us. `/.python-version`
|
||||
# is the single source of truth; tests/test_python_engine_contract.py holds this to it.
|
||||
target-version = "py312"
|
||||
|
||||
[lint]
|
||||
# Beyond the default set, the rules that hold a LOG LINE to what E22.5 makes of it:
|
||||
|
||||
@@ -15,6 +15,7 @@ tests that COMMIT (the whole point of this lane — transaction-ownership behavi
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
@@ -248,6 +249,27 @@ def as_user(db, override_dep, _configured):
|
||||
return _as
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def upload_headers():
|
||||
"""Build the headers `POST /api/downloads/upload` takes since P1.
|
||||
|
||||
Shared, because two test modules drive that route and a second copy of this is exactly the drift
|
||||
that lets the wire shape change in one place only. The body IS the file now; the names ride as
|
||||
percent-encoded headers (`quote` mirrors the client's `encodeURIComponent`, since headers are
|
||||
latin-1 and a Hungarian filename would not survive raw)."""
|
||||
|
||||
def _headers(filename: str, display_name: str | None = None) -> dict[str, str]:
|
||||
headers = {
|
||||
"content-type": "application/octet-stream",
|
||||
"x-upload-filename": quote(filename),
|
||||
}
|
||||
if display_name is not None:
|
||||
headers["x-upload-display-name"] = quote(display_name)
|
||||
return headers
|
||||
|
||||
return _headers
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_rate_limiters():
|
||||
"""Every RateLimiter is a module-level singleton keyed by client IP, and TestClient always
|
||||
|
||||
@@ -100,7 +100,7 @@ def test_the_single_playlist_repull_shares_the_pull_cooldown(db, as_user, make_u
|
||||
db.commit()
|
||||
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: {"ok": True})
|
||||
monkeypatch.setattr(plroutes, "sync_user_playlists", lambda d, u: {"ok": True})
|
||||
monkeypatch.setattr(plroutes, "pull_user_playlists", lambda d, u: {"ok": True})
|
||||
|
||||
c = as_user(user)
|
||||
codes = [c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code for _ in range(4)]
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""E30: an admin action that counts something keeps the count as data in its audit row's `after`, under
|
||||
the key the audit log names in the reader's language (`scheduler.result.<key>`, frontend
|
||||
auditColumns.tsx) — not only in the English `summary`. A renamed key would fall back to the raw word.
|
||||
DB lane (Postgres).
|
||||
"""
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import AuditLog
|
||||
|
||||
|
||||
def _after(db, action: str) -> dict:
|
||||
db.expire_all()
|
||||
return db.execute(select(AuditLog.after).where(AuditLog.action == action)).scalar_one()
|
||||
|
||||
|
||||
def test_clearing_the_audit_log_keeps_the_count_as_data(as_user, db, make_user):
|
||||
admin = make_user(email="boss@example.com", role="admin")
|
||||
db.add_all([AuditLog(action="sync_pause"), AuditLog(action="sync_resume")])
|
||||
db.commit()
|
||||
assert as_user(admin).delete("/api/admin/audit").json()["cleared"] == 2
|
||||
assert _after(db, "audit_clear") == {"cleared": 2}
|
||||
|
||||
|
||||
def test_a_demo_reset_keeps_the_count_as_data(as_user, db, make_user):
|
||||
admin = make_user(email="boss@example.com", role="admin")
|
||||
seeded = as_user(admin).post("/api/admin/demo/reset").json()["playlists_seeded"]
|
||||
assert _after(db, "demo_reset") == {"playlists_seeded": seeded}
|
||||
|
||||
|
||||
def test_a_discovery_purge_keeps_its_counts_as_data(as_user, db, make_user):
|
||||
admin = make_user(email="boss@example.com", role="admin")
|
||||
body = as_user(admin).post("/api/admin/purge-discovery").json()
|
||||
body.pop("ok")
|
||||
assert _after(db, "discovery_purge") == body
|
||||
assert {"videos_deleted", "channels_deleted", "search_videos_deleted"} <= set(body)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""A YouTube topic URL's slug → the auto-tag it gives (pure). The slugs are the ones YouTube actually
|
||||
links (measured on the owner's channels, 2026-09-29): a key spelled differently from YouTube's page
|
||||
never matches, silently — "Humor" sat in the map for months while YouTube links …/wiki/Humour."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.sync.autotag import map_topic_slug
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("slug", "tag"),
|
||||
[
|
||||
("Humour", "Comedy"),
|
||||
("Video_game_culture", "Gaming"),
|
||||
("Electronic_music", "Music"),
|
||||
("Association_football", "Sports"),
|
||||
("Knowledge", "Education"),
|
||||
("Physical_fitness", "Health & Fitness"),
|
||||
("Television_program", "TV"),
|
||||
("Lifestyle_(sociology)", None), # YouTube's catch-all: deliberately no tag
|
||||
],
|
||||
)
|
||||
def test_a_youtube_topic_slug_gives_its_tag(slug, tag):
|
||||
assert map_topic_slug(slug) == tag
|
||||
@@ -0,0 +1,705 @@
|
||||
"""SEC-20 — the global request-body cap (`app/bodylimit.py`).
|
||||
|
||||
Two halves, deliberately kept apart:
|
||||
|
||||
* the MIDDLEWARE tests drive a toy ASGI app, so they pin the mechanism (declared length, counted
|
||||
bytes, the exemption match) with no router, no auth and no database in the way;
|
||||
* the APP tests drive the real application, so they pin that the middleware is actually wired in,
|
||||
sits ahead of the auth dependencies, and refuses in the same shape `capped_body` does.
|
||||
|
||||
The pair that matters most is "an oversized body is refused" together with "a normal body still
|
||||
gets through": the first alone would pass just as well if the cap refused *everything*.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from starlette.requests import Request
|
||||
|
||||
from app import bodylimit, debugsession, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import SystemConfig
|
||||
from app.bodylimit import DEFAULT_MAX_BODY_BYTES, BodyLimitMiddleware
|
||||
from app.routes import diagnostics as diagnostics_routes
|
||||
|
||||
SMALL = b"x" * 16
|
||||
OVER = b"x" * (DEFAULT_MAX_BODY_BYTES + 1)
|
||||
|
||||
|
||||
async def _reading_app(scope, receive, send):
|
||||
"""A toy app that reads the whole body the way a route does, then reports its length.
|
||||
|
||||
It uses a real Starlette `Request`, so the faked `http.disconnect` reaches it as the real
|
||||
ClientDisconnect a real handler would see — the middleware's exception path is exercised, not
|
||||
stubbed around.
|
||||
"""
|
||||
body = await Request(scope, receive).body()
|
||||
payload = json.dumps({"read": len(body)}).encode()
|
||||
await send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": 200,
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
}
|
||||
)
|
||||
await send({"type": "http.response.body", "body": payload})
|
||||
|
||||
|
||||
def _client(**kwargs) -> TestClient:
|
||||
return TestClient(BodyLimitMiddleware(_reading_app, **kwargs))
|
||||
|
||||
|
||||
def _chunks(total: int, size: int = 64 * 1024):
|
||||
"""A body with NO Content-Length: httpx sends an iterator chunked, which is exactly the shape a
|
||||
declared-length check cannot see."""
|
||||
sent = 0
|
||||
while sent < total:
|
||||
n = min(size, total - sent)
|
||||
yield b"x" * n
|
||||
sent += n
|
||||
|
||||
|
||||
# --- the mechanism ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_declared_length_over_the_cap_is_refused_without_reading_the_body():
|
||||
r = _client().post("/anything", content=OVER)
|
||||
assert r.status_code == 413
|
||||
assert r.content == bodylimit.TOO_LARGE_BODY
|
||||
|
||||
|
||||
def test_a_body_under_the_cap_is_passed_through_untouched():
|
||||
# The mutation guard for the test above: if the cap refused everything, this would fail.
|
||||
r = _client().post("/anything", content=SMALL)
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"read": len(SMALL)}
|
||||
|
||||
|
||||
def test_a_chunked_body_is_refused_on_the_COUNTED_bytes():
|
||||
# No Content-Length at all, so the only thing that can stop this is counting as it arrives.
|
||||
r = _client().post("/anything", content=_chunks(DEFAULT_MAX_BODY_BYTES + 64 * 1024))
|
||||
assert r.status_code == 413
|
||||
assert r.content == bodylimit.TOO_LARGE_BODY
|
||||
|
||||
|
||||
def test_a_chunked_body_under_the_cap_still_arrives_whole():
|
||||
size = DEFAULT_MAX_BODY_BYTES - 1
|
||||
r = _client().post("/anything", content=_chunks(size))
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"read": size}
|
||||
|
||||
|
||||
def test_an_understated_content_length_does_not_buy_a_pass():
|
||||
# The declared-length check is the cheap half; a client that lies must still be counted out.
|
||||
r = _client().post("/anything", content=OVER, headers={"content-length": "10"})
|
||||
assert r.status_code == 413
|
||||
|
||||
|
||||
def test_exactly_at_the_cap_is_allowed():
|
||||
size = DEFAULT_MAX_BODY_BYTES
|
||||
r = _client(limit=size).post("/anything", content=b"x" * size)
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"read": size}
|
||||
|
||||
|
||||
# --- the per-route ceilings ---------------------------------------------------------------------
|
||||
|
||||
BIG = {("POST", "/api/things/{tid}/blob"): DEFAULT_MAX_BODY_BYTES * 4}
|
||||
|
||||
|
||||
def test_a_route_with_a_raised_ceiling_may_send_a_body_over_the_global_cap():
|
||||
r = _client(routes=BIG).post("/api/things/abc/blob", content=OVER)
|
||||
assert r.status_code == 200
|
||||
assert r.json() == {"read": len(OVER)}
|
||||
|
||||
|
||||
def test_a_raised_ceiling_is_still_a_ceiling():
|
||||
"""The difference between this and an exemption, and the reason the list is a mapping: a route
|
||||
that needs more bytes does not thereby get unlimited bytes. Left uncounted, the upload route
|
||||
would let an ANONYMOUS caller stream without bound — FastAPI parses a declared body before the
|
||||
first dependency, so `require_human` has not run yet when the bytes land."""
|
||||
over = b"x" * (BIG[("POST", "/api/things/{tid}/blob")] + 1)
|
||||
r = _client(routes=BIG).post("/api/things/abc/blob", content=over)
|
||||
assert r.status_code == 413
|
||||
assert r.content == bodylimit.TOO_LARGE_BODY
|
||||
|
||||
|
||||
def test_a_raised_ceiling_counts_a_chunked_body_too():
|
||||
cap = BIG[("POST", "/api/things/{tid}/blob")]
|
||||
r = _client(routes=BIG).post("/api/things/abc/blob", content=_chunks(cap + 64 * 1024))
|
||||
assert r.status_code == 413
|
||||
|
||||
|
||||
def test_the_raised_ceiling_is_scoped_to_its_method_and_its_path():
|
||||
client = _client(routes=BIG)
|
||||
# Same path, different method.
|
||||
assert client.put("/api/things/abc/blob", content=OVER).status_code == 413
|
||||
# A neighbour that merely starts the same way.
|
||||
assert client.post("/api/things/abc/blob/extra", content=OVER).status_code == 413
|
||||
|
||||
|
||||
def test_a_path_parameter_cannot_swallow_a_slash():
|
||||
# `{tid}` matching greedily across `/` would hand the raised ceiling to routes without one.
|
||||
mw = BodyLimitMiddleware(_reading_app, routes={("POST", "/a/{tid}/b"): 99})
|
||||
assert mw.limit_for("POST", "/a/one/b") == 99
|
||||
assert mw.limit_for("POST", "/a/one/two/b") == DEFAULT_MAX_BODY_BYTES
|
||||
|
||||
|
||||
def test_every_request_gets_a_finite_ceiling():
|
||||
"""There is no pass-through path. A route that is not listed gets the default; one that is
|
||||
listed gets its own; nothing is simply not counted."""
|
||||
mw = BodyLimitMiddleware(_reading_app)
|
||||
assert mw.limit_for("GET", "/api/anything") == DEFAULT_MAX_BODY_BYTES
|
||||
for (method, template), cap in bodylimit.ROUTE_MAX_BODY_BYTES.items():
|
||||
assert cap > DEFAULT_MAX_BODY_BYTES # the admission rule: raised, never lowered
|
||||
assert mw.limit_for(method, template.replace("{sid}", "abc")) == cap
|
||||
|
||||
|
||||
def _declared_routes(root) -> set[tuple[str, str]]:
|
||||
"""Every (method, path template) the app serves.
|
||||
|
||||
It has to RECURSE: since FastAPI 0.141 / Starlette 1.6 an `include_router` leaves an opaque
|
||||
`_IncludedRouter` in `app.routes` and keeps the real routes under its `original_router`, instead
|
||||
of flattening them into the parent as older versions did. Reading `app.routes` alone finds 35
|
||||
entries and none of ours; walking the tree finds all 252.
|
||||
"""
|
||||
found: set[tuple[str, str]] = set()
|
||||
stack = [root]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
for route in getattr(node, "routes", ()) or ():
|
||||
inner = getattr(route, "original_router", None)
|
||||
if inner is not None:
|
||||
stack.append(inner)
|
||||
continue
|
||||
for method in getattr(route, "methods", None) or ():
|
||||
found.add((method, route.path))
|
||||
return found
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("method", "template"), sorted(bodylimit.ROUTE_MAX_BODY_BYTES))
|
||||
def test_every_listed_template_still_resolves_to_a_real_route(method, template):
|
||||
"""The drift guard. The list names route templates that live in another file; a renamed route
|
||||
must fail the gate here, not lose its allowance silently in production."""
|
||||
from app.main import app
|
||||
|
||||
assert (method, template) in _declared_routes(app)
|
||||
|
||||
|
||||
def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes():
|
||||
"""The headroom the module's docblock claims, held as a RATIO rather than as a figure in prose.
|
||||
The client log sink caps itself at 512 KiB — below the default, which is why it is not listed:
|
||||
the global ceiling never fires on it and raising it could only widen what it accepts.
|
||||
|
||||
`<=` alone would not hold the claim. The docblock calls the 2x margin load-bearing precisely
|
||||
because a ceiling that refuses a legitimate request is a worse bug than the one this module
|
||||
fixes, and a margin of 1x is no margin at all.
|
||||
"""
|
||||
assert diagnostics_routes.BODY_MAX_BYTES * 2 <= DEFAULT_MAX_BODY_BYTES
|
||||
assert not any(
|
||||
path.startswith("/api/diagnostics") for _, path in bodylimit.ROUTE_MAX_BODY_BYTES
|
||||
)
|
||||
|
||||
|
||||
# Every body-reading call site in the routers, as (module, function, limit expression). A route that
|
||||
# reads its own body declares the size it MEANS to accept; if that is above the global default and
|
||||
# its route is not in ROUTE_MAX_BODY_BYTES, this middleware refuses a request the route would have
|
||||
# taken — silently, in production, on a path nobody tested. The point of the list is the third entry.
|
||||
#
|
||||
# BOTH helpers, not just `capped_body`: P1 added `capped_stream_to` as its sibling and the registry
|
||||
# did not follow, so the upload route — the one this whole invariant was written for — was the one
|
||||
# site the drift guard could not see (deep-review round 1). The names come from `_common.py` itself
|
||||
# below, so a THIRD sibling cannot be added without this list noticing either — and a route that
|
||||
# skips the helpers altogether and awaits `request.body()` lands here as `<no ceiling declared>`
|
||||
# rather than vanishing (round 3).
|
||||
#
|
||||
# The module is the path relative to `app/routes/`, so a site under `plex/` is spelled
|
||||
# `plex/admin.py` and cannot collide with the top-level `admin.py`.
|
||||
CAPPED_BODY_SITES = {
|
||||
("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"),
|
||||
("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"),
|
||||
("downloads.py", "spool_upload", "capped_stream_to", "cap"),
|
||||
}
|
||||
|
||||
|
||||
def _body_readers() -> set[str]:
|
||||
"""The names of the body-reading helpers `_common.py` exports — asked of the module rather than
|
||||
listed here, so adding a fourth one cannot quietly leave the registry below behind."""
|
||||
common = pathlib.Path(__file__).resolve().parents[1] / "app" / "routes" / "_common.py"
|
||||
tree = ast.parse(common.read_text("utf-8"), filename=str(common))
|
||||
return {
|
||||
node.name
|
||||
for node in tree.body
|
||||
if isinstance(node, (ast.AsyncFunctionDef, ast.FunctionDef))
|
||||
and node.name.startswith("capped_")
|
||||
}
|
||||
|
||||
|
||||
def test_the_registry_covers_every_body_reading_helper():
|
||||
"""The mutation guard for the tests below: if a helper were renamed or added, the site list
|
||||
would still match itself while the new one went uncovered."""
|
||||
assert _body_readers() == {"capped_body", "capped_stream_to"}, (
|
||||
"routes/_common.py grew or renamed a body-reading helper — add its call sites to "
|
||||
"CAPPED_BODY_SITES and this expectation."
|
||||
)
|
||||
|
||||
|
||||
# The ways a route can read its own body WITHOUT one of `_common.py`'s helpers. Every `Request`
|
||||
# parameter under `app/routes/` is named `request`, so the base name is an exact filter and not a
|
||||
# guess — the only other `.json()` in the package is `exc.json()` on a ValidationError.
|
||||
#
|
||||
# Without this set the three guards below know only the `capped_*` naming convention, which means
|
||||
# they are blind to the shape `_common.py` itself is written in. A route spelling `await
|
||||
# request.body()` inline declares no ceiling, hands no connection back, and would be invisible to
|
||||
# all three — reading as "no new sites" rather than as a failure, which is the exact way a drift
|
||||
# guard dies quietly (deep-review round 3).
|
||||
RAW_BODY_READS = {"body", "stream", "form", "json"}
|
||||
|
||||
|
||||
def _call_name(node: ast.Call) -> str | None:
|
||||
if isinstance(node.func, ast.Attribute):
|
||||
return node.func.attr
|
||||
return getattr(node.func, "id", None)
|
||||
|
||||
|
||||
def _body_read_calls(fn: ast.AST, readers: set[str]) -> list[tuple[ast.Call, str]]:
|
||||
"""Every call inside `fn` that reads the request body, as (node, name) in source order."""
|
||||
out: list[tuple[ast.Call, str]] = []
|
||||
for node in ast.walk(fn):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
if (
|
||||
isinstance(node.func, ast.Attribute)
|
||||
and isinstance(node.func.value, ast.Name)
|
||||
and node.func.value.id == "request"
|
||||
and node.func.attr in RAW_BODY_READS
|
||||
):
|
||||
out.append((node, f"request.{node.func.attr}"))
|
||||
elif _call_name(node) in readers:
|
||||
out.append((node, _call_name(node) or ""))
|
||||
return sorted(out, key=lambda pair: (pair[0].lineno, pair[0].col_offset))
|
||||
|
||||
|
||||
def _body_reading_functions() -> dict[tuple[str, str], dict]:
|
||||
"""Every function under `app/routes/` that reads a request body itself, keyed by
|
||||
(module, function).
|
||||
|
||||
Parsed, not grepped. A regex over the call text has to guess how the arguments are spelled, and
|
||||
the shapes it guesses wrong — a bare `4 * 1024 * 1024`, a parameter not named `request` — are
|
||||
the natural ways the NEXT site gets written, so a miss reads as "no new sites" rather than as a
|
||||
failure. `ast` sees every call whatever it looks like.
|
||||
|
||||
Keyed by the ENCLOSING FUNCTION, not just the module: a set keyed by module and limit
|
||||
expression collapses two call sites that spell their limit the same way, which is exactly what
|
||||
a second route copied from the first would do.
|
||||
|
||||
The module half of the key is the path RELATIVE to `app/routes/`, not the basename: the scan
|
||||
recurses, and `admin.py`, `discover.py` and `playlists.py` each exist both at the top level and
|
||||
under `plex/`. Keying by basename would let the second of a colliding pair overwrite the first
|
||||
and take its `releases` flag out of the comparison with it — the same entry-collapse this
|
||||
registry was widened to close.
|
||||
|
||||
`_common.py` is skipped: it is where the helpers live, so its own `request.stream()` loops are
|
||||
the implementation of the rule rather than a violation of it.
|
||||
"""
|
||||
readers = _body_readers()
|
||||
routes_dir = pathlib.Path(__file__).resolve().parents[1] / "app" / "routes"
|
||||
found: dict[tuple[str, str], dict] = {}
|
||||
for py in sorted(routes_dir.rglob("*.py")):
|
||||
if py.name == "_common.py":
|
||||
continue
|
||||
module = py.relative_to(routes_dir).as_posix()
|
||||
tree = ast.parse(py.read_text("utf-8"), filename=str(py))
|
||||
for fn in ast.walk(tree):
|
||||
if not isinstance(fn, (ast.AsyncFunctionDef, ast.FunctionDef)):
|
||||
continue
|
||||
calls = _body_read_calls(fn, readers)
|
||||
if not calls:
|
||||
continue
|
||||
names = [name for _, name in calls]
|
||||
limits = []
|
||||
for node, name in zip([n for n, _ in calls], names):
|
||||
if name.startswith("request."):
|
||||
limits.append("<no ceiling declared>")
|
||||
continue
|
||||
limit = next((kw.value for kw in node.keywords if kw.arg == "limit"), None)
|
||||
if limit is None:
|
||||
pos = 2 if name == "capped_stream_to" else 1
|
||||
limit = node.args[pos] if len(node.args) > pos else None
|
||||
limits.append(ast.unparse(limit) if limit is not None else "<none>")
|
||||
# POSITION, not presence. `any(...)` over the function accepted a release that ran
|
||||
# AFTER the await it exists to precede — the natural spelling for someone reading the
|
||||
# name as "release it when you're done" — while pinning a connection for the whole
|
||||
# transfer. The first body read is what the release has to beat.
|
||||
releases = [
|
||||
node
|
||||
for node in ast.walk(fn)
|
||||
if isinstance(node, ast.Call) and _call_name(node) == "release_read_connection"
|
||||
]
|
||||
first_read = calls[0][0].lineno
|
||||
found[(module, fn.name)] = {
|
||||
"helpers": names,
|
||||
"limits": limits,
|
||||
"releases": any(node.lineno < first_read for node in releases),
|
||||
"releases_late": bool(releases)
|
||||
and not any(node.lineno < first_read for node in releases),
|
||||
}
|
||||
return found
|
||||
|
||||
|
||||
def test_every_body_reading_site_is_known_and_fits_under_its_ceiling():
|
||||
found = {
|
||||
(module, fn, site["helpers"][i], site["limits"][i])
|
||||
for (module, fn), site in _body_reading_functions().items()
|
||||
for i in range(len(site["helpers"]))
|
||||
}
|
||||
assert found == CAPPED_BODY_SITES, (
|
||||
f"a body-reading call site appeared, moved or changed shape: {found ^ CAPPED_BODY_SITES}. "
|
||||
"Each one declares a body size; check it against bodylimit.DEFAULT_MAX_BODY_BYTES and, if "
|
||||
"it is larger, add its route to bodylimit.ROUTE_MAX_BODY_BYTES — then update this list."
|
||||
)
|
||||
# And the known ones really do satisfy the rule.
|
||||
assert diagnostics_routes.BODY_MAX_BYTES <= DEFAULT_MAX_BODY_BYTES
|
||||
assert debugsession.SCREENSHOT_MAX_BYTES > DEFAULT_MAX_BODY_BYTES
|
||||
assert ("POST", "/api/debug-sessions/{sid}/screenshots") in bodylimit.ROUTE_MAX_BODY_BYTES
|
||||
# The upload site's limit is the sysconfig knob, which is bounded BY the edge ceiling rather
|
||||
# than being a constant to compare against — that pairing is its own test below.
|
||||
assert ("POST", "/api/downloads/upload") in bodylimit.ROUTE_MAX_BODY_BYTES
|
||||
|
||||
|
||||
def test_release_read_connection_ends_the_transaction_it_was_called_for(db):
|
||||
"""The helper itself: a SELECT has autobegun, and afterwards it has not."""
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.db import release_read_connection
|
||||
|
||||
db.execute(select(SystemConfig).limit(1))
|
||||
assert db.in_transaction(), "nothing autobegan, so this test would pass vacuously"
|
||||
release_read_connection(db)
|
||||
assert not db.in_transaction()
|
||||
|
||||
|
||||
def test_release_read_connection_refuses_to_discard_pending_work(db):
|
||||
"""The precondition, ENFORCED rather than described.
|
||||
|
||||
The session belongs to the whole request, not to the dependency calling this — so if something
|
||||
ahead of it ever stages a write, ending the transaction would throw that write away with
|
||||
nothing in the log. `service.enqueue` already learned that a whole-session rollback takes the
|
||||
caller's pending work with it, which is why it uses a SAVEPOINT. A comment cannot stop the next
|
||||
dependency from being added in the wrong place; this can.
|
||||
"""
|
||||
from app.db import release_read_connection
|
||||
|
||||
db.add(SystemConfig(key="probe_pending_write", value="1"))
|
||||
with pytest.raises(RuntimeError, match="pending work"):
|
||||
release_read_connection(db)
|
||||
db.rollback() # the staged row is this test's own
|
||||
|
||||
|
||||
def test_release_read_connection_refuses_to_discard_a_FLUSHED_write(db):
|
||||
"""The half the unit-of-work check cannot see.
|
||||
|
||||
`db.new`/`dirty`/`deleted` empty out the moment a helper flushes, and CLAUDE.md's canonical
|
||||
split is "the handler commits; helpers flush" — so the single most likely future violator is
|
||||
the one shape the obvious check is blind to. Measured before the fix: all three sets empty, the
|
||||
guard silent, the row gone after the rollback.
|
||||
"""
|
||||
from app.db import release_read_connection
|
||||
|
||||
db.add(SystemConfig(key="probe_flushed_write", value="1"))
|
||||
db.flush()
|
||||
assert not (db.new or db.dirty or db.deleted), (
|
||||
"the flush did not clear the unit-of-work, so this test would pass for the wrong reason"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="already WRITTEN"):
|
||||
release_read_connection(db)
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_release_read_connection_refuses_to_discard_a_core_dml_write(db):
|
||||
"""The other half: `db.execute(update(...))` never enters the unit-of-work at all, and
|
||||
`downloads/service.py` writes that way on purpose (a conditional `UPDATE … WHERE status=…` is
|
||||
the honest-states pattern)."""
|
||||
from sqlalchemy import insert
|
||||
|
||||
from app.db import release_read_connection
|
||||
|
||||
db.execute(insert(SystemConfig).values(key="probe_core_dml", value="1"))
|
||||
assert not (db.new or db.dirty or db.deleted), (
|
||||
"Core DML entered the unit-of-work, so this test would pass for the wrong reason"
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="already WRITTEN"):
|
||||
release_read_connection(db)
|
||||
db.rollback()
|
||||
|
||||
|
||||
def test_every_body_reading_site_hands_its_pooled_connection_back_first():
|
||||
"""Reading the body yourself is what makes a request SLOW inside its own database session.
|
||||
|
||||
The auth (or ownership) dependency that must resolve first is a SELECT, so SQLAlchemy has
|
||||
autobegun and the session is holding one of the pool's connections; awaiting the body then pins
|
||||
it for as long as the client keeps sending — one connection per in-flight upload, `idle in
|
||||
transaction`, with nothing to time it out. `db.release_read_connection` ends that transaction,
|
||||
and every one of these sites needs it.
|
||||
|
||||
Held generically because the three sites arrived one at a time and only the newest was fixed:
|
||||
a fourth body-reading route gets the same requirement automatically.
|
||||
|
||||
BEFORE the read, not merely somewhere in the function. A release that runs after the await
|
||||
satisfies the letter of the rule and none of its purpose — the connection was held for exactly
|
||||
the span the rule exists to protect — so `releases_late` is reported as its own, louder case.
|
||||
"""
|
||||
sites = _body_reading_functions()
|
||||
late = [f"{module}:{fn}" for (module, fn), site in sites.items() if site["releases_late"]]
|
||||
missing = [
|
||||
f"{module}:{fn}"
|
||||
for (module, fn), site in sites.items()
|
||||
if not site["releases"] and not site["releases_late"]
|
||||
]
|
||||
assert not late, (
|
||||
"a route calls db.release_read_connection AFTER it reads the body, which holds the pooled "
|
||||
f"connection for exactly the transfer it was supposed to free: {late}. Move the call above "
|
||||
"the await."
|
||||
)
|
||||
assert not missing, (
|
||||
"a route reads its own request body while still inside its read transaction, pinning a "
|
||||
f"pooled connection for the whole transfer: {missing}. Call "
|
||||
"db.release_read_connection(db) before the await — see routes/downloads.py:spool_upload."
|
||||
)
|
||||
|
||||
|
||||
def test_the_upload_ceiling_and_the_admin_knob_cannot_drift_apart():
|
||||
"""The upload route's in-handler cap is a `sysconfig` value an admin can change; the edge is a
|
||||
constant. If the knob could be set above the edge, an admin raising it would silently start
|
||||
getting 413s from a layer the admin page never mentions.
|
||||
|
||||
Since P1 the two count the SAME bytes — the body is the file, with no multipart envelope
|
||||
wrapped round it — so the old allowance is gone and the edge is the knob's own maximum,
|
||||
exactly. A gap reappearing means someone re-wrapped the upload in an envelope without saying so.
|
||||
|
||||
Equality also means the EDGE is what answers on a default install, where no override has lowered
|
||||
the knob: the declared-length check refuses before a byte is read. That is not a gap to close
|
||||
with headroom (measured: headroom only moves the boundary by its own size, and the route's drain
|
||||
pushes the count back over it) — it is a refusal the client has to be able to read, which is why
|
||||
`errors.tooLarge` exists and why the dialog checks the file against `upload_max_bytes` first.
|
||||
"""
|
||||
spec = next(s for s in sysconfig.SPECS if s.key == "download_upload_max_bytes")
|
||||
edge = bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/downloads/upload")]
|
||||
assert spec.max == edge
|
||||
|
||||
|
||||
def _route_objects(root):
|
||||
"""`_declared_routes`' sibling, keeping the route OBJECT so its FastAPI dependant can be read."""
|
||||
found: dict[tuple[str, str], object] = {}
|
||||
stack = [root]
|
||||
while stack:
|
||||
node = stack.pop()
|
||||
for route in getattr(node, "routes", ()) or ():
|
||||
inner = getattr(route, "original_router", None)
|
||||
if inner is not None:
|
||||
stack.append(inner)
|
||||
continue
|
||||
for method in getattr(route, "methods", None) or ():
|
||||
found[(method, route.path)] = route
|
||||
return found
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("method", "template"), sorted(bodylimit.ROUTE_MAX_BODY_BYTES))
|
||||
def test_no_route_with_a_raised_ceiling_declares_a_body_parameter(method, template):
|
||||
"""P1's invariant, and the reason this is a test rather than a comment.
|
||||
|
||||
A raised ceiling is only honest when the route's own cap applies to the bytes AS THEY ARRIVE.
|
||||
FastAPI reads and spools a DECLARED body (`= File(...)`, `= Form(...)`, a Pydantic model)
|
||||
BEFORE the first dependency resolves — so on such a route this middleware's ceiling is the only
|
||||
thing between an anonymous caller and the whole allowance, `require_human` never gets a say, and
|
||||
the route's in-handler cap measures bytes that have already landed. That is exactly what P1
|
||||
closed on the upload route, which sat in `ROUTE_MAX_BODY_BYTES` with a 500 MiB ceiling and a
|
||||
`file: UploadFile = File(...)` body for one release.
|
||||
|
||||
Asked of FastAPI itself rather than parsed or grepped: the router already knows which parameters
|
||||
it will read the body for, and a shape a regex or an AST walk guesses wrong is precisely how the
|
||||
next route slips through.
|
||||
|
||||
`route.body_field`, NOT `route.dependant.body_params`, and the difference is the whole test.
|
||||
`body_params` holds only the parameters of the route's OWN callable; `body_field` is built from
|
||||
`_get_flat_body_params`, which recurses into sub-dependencies — and that is also what FastAPI
|
||||
itself consults when it decides to read the body. Measured on the pinned FastAPI: a body declared
|
||||
inside a sub-dependency leaves `body_params` EMPTY while the body is still read in full before
|
||||
auth, so the first version of this test was green over a reproduction of the very defect it
|
||||
exists to forbid (deep-review round 1). The upload route's own idiom — a body read from a
|
||||
`yield` dependency — is exactly the shape that would have slipped through."""
|
||||
from app.main import app
|
||||
|
||||
route = _route_objects(app).get((method, template))
|
||||
assert route is not None, f"{method} {template} is not a route any more"
|
||||
assert route.body_field is None, (
|
||||
f"{method} {template} declares a body ({route.body_field.name}) on the route or in one of "
|
||||
"its dependencies, so FastAPI reads it before the first dependency runs and the raised "
|
||||
"ceiling is all that stands between an anonymous caller and the whole allowance. Read the "
|
||||
"body yourself, from a dependency declared AFTER the auth one — see "
|
||||
"routes/downloads.py:spool_upload."
|
||||
)
|
||||
|
||||
|
||||
def test_a_stored_override_outside_the_bounds_falls_back_on_READ(db):
|
||||
"""`set_value` refuses an out-of-range int, but neither bound has always been what it is now —
|
||||
the upload knob had no `max` before SEC-20 and its `min` was 0 before E7 — so a value that was
|
||||
legal when it was written can be sitting in the database. Served as-is it is exactly the drift
|
||||
the test above forbids: the route would accept bytes the edge refuses.
|
||||
|
||||
The fallback is the env default and a warning, NOT a clamp. A clamp is a silent rewrite, and on
|
||||
the min side it would be a wrong one: a stored 0 meant "no per-file cap" to the route, and
|
||||
clamped to 1 it becomes a one-byte cap that kills uploads with nothing in the log.
|
||||
"""
|
||||
spec = next(s for s in sysconfig.SPECS if s.key == "download_upload_max_bytes")
|
||||
assert spec.max is not None and spec.min is not None
|
||||
default = settings.download_upload_max_bytes
|
||||
|
||||
row = SystemConfig(key="download_upload_max_bytes", value=str(spec.max * 4))
|
||||
db.add(row)
|
||||
db.commit()
|
||||
assert sysconfig.get_int(db, "download_upload_max_bytes") == default
|
||||
|
||||
row.value = "0" # the legacy "no per-file cap", now below min
|
||||
db.commit()
|
||||
assert sysconfig.get_int(db, "download_upload_max_bytes") == default
|
||||
|
||||
# An override that IS in range must still be honoured — otherwise a bounds predicate written
|
||||
# backwards would make every override inert and the two assertions above could not tell.
|
||||
# Deliberately not `spec.max`: that equals the env default for this key, so the assertion would
|
||||
# be satisfied by the fallback it is meant to rule out.
|
||||
row.value = str(spec.max - 1)
|
||||
db.commit()
|
||||
assert spec.max - 1 != default
|
||||
assert sysconfig.get_int(db, "download_upload_max_bytes") == spec.max - 1
|
||||
|
||||
|
||||
def test_the_cap_sits_inside_the_response_decorators_and_outside_the_setup_gate():
|
||||
"""The registration order, held in code rather than in the comment beside it. `add_middleware`
|
||||
inverts the reading order (last registered = OUTERMOST), which is exactly the kind of thing a
|
||||
prose description gets backwards — it already did once.
|
||||
|
||||
The layers are named, not counted by class: all four `@app.middleware("http")` decorators —
|
||||
including the setup gate, which must be INSIDE — register as the same `BaseHTTPMiddleware`, so a
|
||||
class-shaped assertion passes for every arrangement anyone would plausibly get wrong.
|
||||
|
||||
Outside the cap: only response decorators, so the 413 is dressed like any other answer. Inside
|
||||
it: everything that does work on the caller's behalf — the setup gate opens a DB session per
|
||||
request while an instance is unconfigured, which is the anonymous pre-auth work SEC-20 exists to
|
||||
remove.
|
||||
"""
|
||||
from app.main import app
|
||||
|
||||
def layer(m) -> str:
|
||||
dispatch = m.kwargs.get("dispatch") if m.kwargs else None
|
||||
return dispatch.__name__ if dispatch is not None else getattr(m.cls, "__name__", str(m.cls))
|
||||
|
||||
names = [layer(m) for m in app.user_middleware] # outer → inner
|
||||
i = names.index("BodyLimitMiddleware")
|
||||
assert set(names[:i]) <= {
|
||||
"log_requests",
|
||||
"security_headers",
|
||||
"static_cache_headers",
|
||||
"CORSMiddleware",
|
||||
}, f"something that is not a response decorator sits outside the cap: {names[:i]}"
|
||||
assert "log_requests" in names[:i], "the access log must stay outside, so a 413 is logged"
|
||||
assert "setup_gate" in names[i + 1 :], "the setup gate must not run for an oversized body"
|
||||
assert "SessionMiddleware" in names[i + 1 :]
|
||||
|
||||
|
||||
def test_the_screenshot_ceiling_matches_the_cap_the_route_itself_enforces():
|
||||
assert debugsession.SCREENSHOT_MAX_BYTES > DEFAULT_MAX_BODY_BYTES
|
||||
assert (
|
||||
bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/debug-sessions/{sid}/screenshots")]
|
||||
== debugsession.SCREENSHOT_MAX_BYTES
|
||||
)
|
||||
|
||||
|
||||
# --- the real application -----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_global_refusal_carries_the_baseline_security_headers():
|
||||
"""The 413 is a response like any other, so it has to be dressed like one. The middleware sits
|
||||
INSIDE the security-header and CORS layers for exactly this reason: a refusal a browser drops
|
||||
for a missing header is indistinguishable from a network failure, and the user would be shown a
|
||||
generic error instead of the translated `too_large`.
|
||||
|
||||
A bare TestClient on purpose, not the `client` fixture: this is the only test that exercises the
|
||||
middleware ORDER, and that fixture is DB-gated — so on a host without Postgres the one assertion
|
||||
holding the move would have skipped while the file still reported green. The refusal never
|
||||
reaches a database; it is answered above the router.
|
||||
"""
|
||||
from app.main import app
|
||||
|
||||
r = TestClient(app).post(
|
||||
"/auth/password-login", content=OVER, headers={"content-type": "text/plain"}
|
||||
)
|
||||
assert r.status_code == 413
|
||||
assert r.headers.get("x-content-type-options") == "nosniff"
|
||||
assert "referrer-policy" in r.headers
|
||||
|
||||
|
||||
def test_the_login_route_is_refused_before_a_single_dependency_runs(client, monkeypatch):
|
||||
"""`/auth/password-login` is the reason SEC-20 exists: it is anonymous, it declares a body, and
|
||||
FastAPI reads that body in full before the first dependency. `_login_limiter.allow` is the very
|
||||
first statement of the handler, so it is the sharpest available proof that nothing ran."""
|
||||
from app.routes import auth as auth_routes
|
||||
|
||||
reached = False
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
nonlocal reached
|
||||
reached = True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(auth_routes._login_limiter, "allow", _record)
|
||||
|
||||
r = client.post("/auth/password-login", content=OVER, headers={"content-type": "text/plain"})
|
||||
assert r.status_code == 413
|
||||
assert not reached, "the handler ran — the cap is inside the router instead of ahead of it"
|
||||
|
||||
|
||||
def test_a_normal_login_body_still_reaches_the_handler(client, monkeypatch):
|
||||
"""The mutation check for the test above. If the cap refused every body, the 413 test would
|
||||
pass while the app was broken; this one would not."""
|
||||
from app.routes import auth as auth_routes
|
||||
|
||||
reached = False
|
||||
|
||||
def _record(*args, **kwargs):
|
||||
nonlocal reached
|
||||
reached = True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(auth_routes._login_limiter, "allow", _record)
|
||||
|
||||
r = client.post("/auth/password-login", json={"email": "nobody@example.com", "password": "x"})
|
||||
assert r.status_code == 401
|
||||
assert reached
|
||||
|
||||
|
||||
def test_the_global_refusal_is_byte_identical_to_the_per_route_one(db, as_user, make_user):
|
||||
"""Two shapes for one fact is how the `cancelled`/`canceled` bug happened. The log sink caps
|
||||
itself with `capped_body`; its 413 and the middleware's must be the same bytes.
|
||||
|
||||
Signed in on purpose: `current_user` resolves before the body dependency, so an anonymous call
|
||||
would be answered 401 and never reach the per-route cap this compares against. The body is over
|
||||
the SINK's 512 KiB but under the global 1 MiB, so it is `capped_body` that refuses it.
|
||||
"""
|
||||
user = make_user()
|
||||
db.commit()
|
||||
over_the_sink = b"x" * (diagnostics_routes.BODY_MAX_BYTES + 1)
|
||||
assert len(over_the_sink) < DEFAULT_MAX_BODY_BYTES
|
||||
r = as_user(user).post(
|
||||
"/api/diagnostics/events",
|
||||
content=over_the_sink,
|
||||
headers={"content-type": "application/json"},
|
||||
)
|
||||
assert r.status_code == 413
|
||||
# RAW BYTES, not parsed JSON: the two used to differ by the two spaces `json.dumps` puts in by
|
||||
# default and `JSONResponse` does not, and a structural comparison could not see it — which is
|
||||
# the only kind of drift the docblock's promise was ever at risk from.
|
||||
assert r.content == bodylimit.TOO_LARGE_BODY
|
||||
@@ -0,0 +1,143 @@
|
||||
"""A channel's metadata is fetched again once it is stale, not only the first time it is seen.
|
||||
|
||||
Banners, avatars, descriptions and counts change on YouTube, and an old banner URL stops resolving:
|
||||
a channel that kept the metadata of the day it was first imported showed a dead banner forever. The
|
||||
subscription import and the channel page both refresh what is older than `DETAILS_MAX_AGE`, and
|
||||
leave fresh metadata alone (each refresh spends quota). DB lane.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models import Channel, Subscription
|
||||
from app.routes import channels
|
||||
from app.sync import subscriptions
|
||||
from app.sync.subscriptions import DETAILS_MAX_AGE, import_subscriptions
|
||||
|
||||
NOW = datetime.now(timezone.utc)
|
||||
FRESH = NOW - timedelta(days=1)
|
||||
STALE = NOW - DETAILS_MAX_AGE - timedelta(hours=1)
|
||||
|
||||
|
||||
def _item(cid: str) -> dict:
|
||||
return {
|
||||
"id": cid,
|
||||
"snippet": {"title": f"{cid} renamed", "publishedAt": "2010-01-01T00:00:00Z"},
|
||||
"brandingSettings": {"image": {"bannerExternalUrl": f"https://yt3.example/{cid}-new"}},
|
||||
}
|
||||
|
||||
|
||||
def _youtube(asked: list[list[str]], subscribed: list[str], gone: tuple[str, ...] = ()):
|
||||
class _Client:
|
||||
def __init__(self, db, user):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def iter_subscriptions(self):
|
||||
for cid in subscribed:
|
||||
yield {"channel_id": cid, "title": cid, "yt_subscription_id": f"sub-{cid}"}
|
||||
|
||||
def get_channels(self, ids):
|
||||
asked.append(sorted(ids))
|
||||
# A terminated / deleted channel is simply absent from `channels.list`'s answer.
|
||||
return [_item(cid) for cid in ids if cid not in gone]
|
||||
|
||||
return _Client
|
||||
|
||||
|
||||
def _channel(db, cid: str, synced: datetime | None) -> None:
|
||||
db.add(
|
||||
Channel(
|
||||
id=cid,
|
||||
title=cid,
|
||||
banner_url=f"https://yt3.example/{cid}-old",
|
||||
published_at=datetime(2010, 1, 1, tzinfo=timezone.utc),
|
||||
details_synced_at=synced,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_the_import_refreshes_stale_and_missing_details_and_leaves_fresh_ones(
|
||||
db, make_user, monkeypatch
|
||||
):
|
||||
user = make_user()
|
||||
_channel(db, "UCfresh", FRESH)
|
||||
_channel(db, "UCstale", STALE)
|
||||
_channel(db, "UCnever", None)
|
||||
db.flush()
|
||||
for cid in ("UCfresh", "UCstale", "UCnever"):
|
||||
db.add(Subscription(user_id=user.id, channel_id=cid))
|
||||
db.commit()
|
||||
asked: list[list[str]] = []
|
||||
monkeypatch.setattr(
|
||||
subscriptions, "YouTubeClient", _youtube(asked, ["UCfresh", "UCstale", "UCnever"])
|
||||
)
|
||||
|
||||
result = import_subscriptions(db, user)
|
||||
|
||||
assert asked == [["UCnever", "UCstale"]]
|
||||
assert result["channels_detailed"] == 2
|
||||
stale = db.get(Channel, "UCstale")
|
||||
assert stale is not None
|
||||
assert stale.banner_url == "https://yt3.example/UCstale-new"
|
||||
assert stale.details_synced_at is not None and stale.details_synced_at > STALE
|
||||
fresh = db.get(Channel, "UCfresh")
|
||||
assert fresh is not None and fresh.banner_url == "https://yt3.example/UCfresh-old"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("synced", "refreshed"), [(STALE, True), (FRESH, False)])
|
||||
def test_the_channel_page_refreshes_a_stale_channel_only(
|
||||
db, make_user, monkeypatch, synced, refreshed
|
||||
):
|
||||
user = make_user()
|
||||
_channel(db, "UCpage", synced)
|
||||
db.commit()
|
||||
asked: list[list[str]] = []
|
||||
monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, []))
|
||||
|
||||
body = channels.channel_detail("UCpage", user=user, db=db)
|
||||
|
||||
assert asked == ([["UCpage"]] if refreshed else [])
|
||||
suffix = "new" if refreshed else "old"
|
||||
assert body["banner_url"] == f"https://yt3.example/UCpage-{suffix}"
|
||||
|
||||
|
||||
def test_a_channel_youtube_no_longer_returns_is_not_asked_for_on_every_view(
|
||||
db, make_user, monkeypatch
|
||||
):
|
||||
"""A terminated channel is missing from `channels.list`'s answer, so nothing was stamped and the
|
||||
channel page asked for it again (1 quota unit) on every single view. The miss is stamped like a
|
||||
fetch, so the next ask waits for `DETAILS_MAX_AGE` like any other channel."""
|
||||
user = make_user()
|
||||
db.add(Channel(id="UCgone", title="UCgone"))
|
||||
db.commit()
|
||||
asked: list[list[str]] = []
|
||||
monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, [], gone=("UCgone",)))
|
||||
|
||||
channels.channel_detail("UCgone", user=user, db=db)
|
||||
channels.channel_detail("UCgone", user=user, db=db)
|
||||
|
||||
assert asked == [["UCgone"]]
|
||||
|
||||
|
||||
def test_the_import_stamps_a_channel_youtube_no_longer_returns(db, make_user, monkeypatch):
|
||||
user = make_user()
|
||||
_channel(db, "UCgone", STALE)
|
||||
db.flush()
|
||||
db.add(Subscription(user_id=user.id, channel_id="UCgone"))
|
||||
db.commit()
|
||||
asked: list[list[str]] = []
|
||||
monkeypatch.setattr(
|
||||
subscriptions, "YouTubeClient", _youtube(asked, ["UCgone"], gone=("UCgone",))
|
||||
)
|
||||
|
||||
import_subscriptions(db, user)
|
||||
import_subscriptions(db, user)
|
||||
|
||||
assert asked == [["UCgone"]]
|
||||
@@ -0,0 +1,302 @@
|
||||
"""E30 S3: every A→Z list the server orders is in the reader's alphabet. The database image is Alpine,
|
||||
whose `en_US.utf8` is BYTE order: "Életrajzi" and "Ötödik" sorted after "Zorró" in the feed, the channel
|
||||
list, the Plex grid, the genre list. Hungarian puts "Ö" after "O" and "Cs" after "Cu". Needs the DB."""
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.collation import sorted_words
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import (
|
||||
Channel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
PlexCollection,
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
Subscription,
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from datetime import datetime, timezone
|
||||
|
||||
WORDS = ["Zorró", "Életrajzi", "Ötödik", "oroszlán", "Csillag", "Cukor"]
|
||||
HU = ["Cukor", "Csillag", "Életrajzi", "oroszlán", "Ötödik", "Zorró"]
|
||||
EN = ["Csillag", "Cukor", "Életrajzi", "oroszlán", "Ötödik", "Zorró"]
|
||||
|
||||
|
||||
def _user(db, lang: str) -> User:
|
||||
user = User(email=f"collate-{lang}@example.com", preferences={"language": lang})
|
||||
db.add(user)
|
||||
db.flush()
|
||||
return user
|
||||
|
||||
|
||||
def _client(db, user, monkeypatch, override_dep) -> TestClient:
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: user)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def test_words_sort_in_the_readers_alphabet(db):
|
||||
assert sorted_words(db, WORDS, "hu") == HU
|
||||
assert sorted_words(db, WORDS, "en") == EN
|
||||
assert sorted_words(db, WORDS, "xx") == EN # an unknown language sorts as English
|
||||
assert sorted_words(db, [], "hu") == []
|
||||
|
||||
|
||||
def _seed_feed(db, user):
|
||||
db.add(Channel(id="UCc", title="Channel"))
|
||||
db.flush()
|
||||
db.add(Subscription(user_id=user.id, channel_id="UCc", priority=0, hidden=False))
|
||||
db.add_all(
|
||||
Video(
|
||||
id=f"v{i}",
|
||||
channel_id="UCc",
|
||||
published_at=datetime(2024, 1, i + 1, tzinfo=timezone.utc),
|
||||
title=w,
|
||||
is_short=False,
|
||||
live_status="none",
|
||||
via_search=False,
|
||||
via_explore=False,
|
||||
)
|
||||
for i, w in enumerate(WORDS)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_the_feed_title_sort_pages_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||
user = _user(db, "hu")
|
||||
_seed_feed(db, user)
|
||||
client = _client(db, user, monkeypatch, override_dep)
|
||||
|
||||
full = [it["title"] for it in client.get("/api/feed?sort=title&limit=50").json()["items"]]
|
||||
assert full == HU
|
||||
# The cursor compares with the same collated expression, so paging walks the same order.
|
||||
walked: list[str] = []
|
||||
cursor = None
|
||||
for _ in range(len(WORDS)):
|
||||
url = "/api/feed?sort=title&limit=2" + (f"&cursor={cursor}" if cursor else "")
|
||||
body = client.get(url).json()
|
||||
walked += [it["title"] for it in body["items"]]
|
||||
cursor = body["next_cursor"]
|
||||
if not body["has_more"]:
|
||||
break
|
||||
assert walked == HU
|
||||
desc = client.get("/api/feed?sort=title_desc&limit=50").json()["items"]
|
||||
assert [it["title"] for it in desc] == HU[::-1]
|
||||
|
||||
|
||||
def test_the_channel_list_is_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||
user = _user(db, "hu")
|
||||
db.add_all(Channel(id=f"UC{i}", title=w) for i, w in enumerate(WORDS))
|
||||
db.flush()
|
||||
db.add_all(
|
||||
Subscription(user_id=user.id, channel_id=f"UC{i}", priority=0, hidden=False)
|
||||
for i in range(len(WORDS))
|
||||
)
|
||||
db.commit()
|
||||
r = _client(db, user, monkeypatch, override_dep).get("/api/channels")
|
||||
assert [c["title"] for c in r.json()] == HU
|
||||
|
||||
|
||||
def _seed_plex(db):
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
col = PlexCollection(rating_key="col1", library_id=lib.id, title="Set", child_count=len(WORDS))
|
||||
db.add(col)
|
||||
db.add_all(
|
||||
PlexItem(
|
||||
rating_key=f"m{i}",
|
||||
library_id=lib.id,
|
||||
kind="movie",
|
||||
title=f"EN {i}",
|
||||
title_hu=w,
|
||||
year=2000,
|
||||
genres_hu=[w],
|
||||
collection_keys=["col1"],
|
||||
)
|
||||
for i, w in enumerate(WORDS)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _plex(db, monkeypatch, override_dep) -> TestClient:
|
||||
user = _user(db, "hu")
|
||||
_seed_plex(db)
|
||||
return _client(db, user, monkeypatch, override_dep)
|
||||
|
||||
|
||||
def test_the_plex_grid_sorts_by_the_hungarian_title(db, monkeypatch, override_dep):
|
||||
client = _plex(db, monkeypatch, override_dep)
|
||||
grid = client.get("/api/plex/library?scope=movie&sort=title&sort_dir=asc&lang=hu").json()
|
||||
assert [it["title"] for it in grid["items"]] == HU
|
||||
|
||||
|
||||
def test_the_plex_genre_list_is_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||
client = _plex(db, monkeypatch, override_dep)
|
||||
facets = client.get("/api/plex/facets?scope=movie&lang=hu").json()
|
||||
assert [g["value"] for g in facets["genres"]] == HU
|
||||
|
||||
|
||||
def test_a_collection_sorts_by_the_hungarian_title(db, monkeypatch, override_dep):
|
||||
client = _plex(db, monkeypatch, override_dep)
|
||||
members = client.get("/api/plex/collections/col1?sort=title&sort_dir=asc&lang=hu").json()
|
||||
assert [it["title"] for it in members["items"]] == HU
|
||||
# year ties everywhere, so the default (year) order falls through to the title tiebreak
|
||||
default = client.get("/api/plex/collections/col1?lang=hu").json()
|
||||
assert [it["title"] for it in default["items"]] == HU
|
||||
|
||||
|
||||
def _seed_discovery(db, user, synced: bool):
|
||||
"""Channels the user's playlist holds one video from each, not subscribed: discovery lists them,
|
||||
count-tied, so the title decides. `synced=False` leaves them stubs, which sends the endpoint down
|
||||
its enrich path — the one that re-sorts in Python and has to agree with the ORDER BY."""
|
||||
pl = Playlist(user_id=user.id, name="Mix")
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
stamp = datetime(2024, 1, 1, tzinfo=timezone.utc) if synced else None
|
||||
for i, w in enumerate(WORDS):
|
||||
db.add(Channel(id=f"UCd{i}", title=w, details_synced_at=stamp))
|
||||
db.flush()
|
||||
db.add(
|
||||
Video(
|
||||
id=f"d{i}",
|
||||
channel_id=f"UCd{i}",
|
||||
published_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
title="x",
|
||||
is_short=False,
|
||||
live_status="none",
|
||||
via_search=False,
|
||||
via_explore=False,
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=f"d{i}", position=i))
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_channel_discovery_is_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||
user = _user(db, "hu")
|
||||
_seed_discovery(db, user, synced=True)
|
||||
r = _client(db, user, monkeypatch, override_dep).get("/api/channels/discovery")
|
||||
assert [c["title"] for c in r.json()] == HU
|
||||
|
||||
|
||||
def test_channel_discovery_keeps_that_order_after_enriching(db, monkeypatch, override_dep):
|
||||
import app.routes.channels as channels
|
||||
|
||||
class FakeYouTube:
|
||||
def __init__(self, *a, **k): ...
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def get_channels(self, ids):
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(channels, "YouTubeClient", FakeYouTube)
|
||||
monkeypatch.setattr(channels, "apply_channel_details", lambda *a, **k: None)
|
||||
user = _user(db, "hu")
|
||||
_seed_discovery(db, user, synced=False)
|
||||
r = _client(db, user, monkeypatch, override_dep).get("/api/channels/discovery")
|
||||
assert [c["title"] for c in r.json()] == HU
|
||||
|
||||
|
||||
# --- The other lists the server puts in the reader's alphabet (E30 review): each call site, once. ---
|
||||
|
||||
|
||||
def _others(db, n_offset: int = 0) -> list[User]:
|
||||
users = [
|
||||
User(email=f"{w.lower()}@example.com", display_name=w) for w in WORDS
|
||||
] # emails and names in WORDS' order
|
||||
db.add_all(users)
|
||||
db.flush()
|
||||
return users
|
||||
|
||||
|
||||
def test_the_share_and_message_recipients_are_in_the_readers_alphabet(
|
||||
db, monkeypatch, override_dep
|
||||
):
|
||||
from app.models import MessageKey
|
||||
|
||||
user = _user(db, "hu")
|
||||
others = _others(db)
|
||||
db.add_all(
|
||||
MessageKey(
|
||||
user_id=u.id,
|
||||
public_key="k",
|
||||
wrapped_private_key="w",
|
||||
salt="s",
|
||||
wrap_iv="i",
|
||||
key_check="c",
|
||||
)
|
||||
for u in others
|
||||
)
|
||||
db.commit()
|
||||
client = _client(db, user, monkeypatch, override_dep)
|
||||
emails = [r["email"] for r in client.get("/api/downloads/recipients").json()]
|
||||
assert emails == [f"{w.lower()}@example.com" for w in HU]
|
||||
names = [r["name"] for r in client.get("/api/messages/users").json()]
|
||||
assert names == HU
|
||||
|
||||
|
||||
def test_the_blocked_channels_are_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||
from app.models import BlockedChannel
|
||||
|
||||
user = _user(db, "hu")
|
||||
db.add_all(Channel(id=f"UC{i}", title=w) for i, w in enumerate(WORDS))
|
||||
db.flush()
|
||||
db.add_all(BlockedChannel(user_id=user.id, channel_id=f"UC{i}") for i in range(len(WORDS)))
|
||||
db.commit()
|
||||
r = _client(db, user, monkeypatch, override_dep).get("/api/channels/blocked/list")
|
||||
assert [c["title"] for c in r.json()] == HU
|
||||
|
||||
|
||||
def test_the_plex_libraries_and_collections_are_in_the_readers_alphabet(
|
||||
db, monkeypatch, override_dep
|
||||
):
|
||||
user = _user(db, "hu")
|
||||
libs = [
|
||||
PlexLibrary(plex_key=str(i), title=w, kind="movie", enabled=True)
|
||||
for i, w in enumerate(WORDS)
|
||||
]
|
||||
db.add_all(libs)
|
||||
db.flush()
|
||||
db.add_all(
|
||||
PlexCollection(rating_key=f"c{i}", library_id=libs[0].id, title=w, child_count=1)
|
||||
for i, w in enumerate(WORDS)
|
||||
)
|
||||
db.commit()
|
||||
client = _client(db, user, monkeypatch, override_dep)
|
||||
assert [lib["title"] for lib in client.get("/api/plex/libraries").json()["libraries"]] == HU
|
||||
cols = client.get("/api/plex/collections").json()["collections"]
|
||||
assert [c["title"] for c in cols] == HU
|
||||
|
||||
|
||||
def test_a_persons_titles_of_one_year_are_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||
user = _user(db, "hu")
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
db.add_all(
|
||||
PlexItem(
|
||||
rating_key=f"m{i}",
|
||||
library_id=lib.id,
|
||||
kind="movie",
|
||||
title=f"EN {i}",
|
||||
title_hu=w,
|
||||
year=2000,
|
||||
cast_names=["Anna"],
|
||||
)
|
||||
for i, w in enumerate(WORDS)
|
||||
)
|
||||
db.commit()
|
||||
body = _client(db, user, monkeypatch, override_dep).get("/api/plex/person/Anna?lang=hu").json()
|
||||
assert [m["title"] for m in body["movies"]] == HU
|
||||
@@ -786,6 +786,7 @@ class FakeGitea:
|
||||
self.redirect = False # an http:// address behind a proxy that sends it to https
|
||||
self.garbage_asset = False # a 2xx that is not the API's answer (a proxy's page)
|
||||
self.patched_body: str | None = None
|
||||
self.markdown_payloads: list[dict] = [] # what the preview asked the tracker to render
|
||||
|
||||
def handler(self, request: httpx.Request) -> httpx.Response:
|
||||
path = request.url.path.removeprefix("/api/v1")
|
||||
@@ -826,6 +827,7 @@ class FakeGitea:
|
||||
self.patched_body = json.loads(request.content)["body"]
|
||||
return httpx.Response(201, json={"number": 7})
|
||||
if request.method == "POST" and path == "/markdown":
|
||||
self.markdown_payloads.append(json.loads(request.content))
|
||||
return httpx.Response(200, text="<h3>Steps</h3>")
|
||||
return httpx.Response(404, json={"message": "not found"})
|
||||
|
||||
@@ -1180,6 +1182,20 @@ def test_the_preview_is_the_trackers_own_rendering(gitea_fake, db, make_user, as
|
||||
assert stranger.post(f"/api/debug-sessions/{sid}/issue").status_code == 404
|
||||
|
||||
|
||||
def test_the_preview_never_asks_the_tracker_for_a_repository_context(
|
||||
gitea_fake, db, make_user, as_user
|
||||
):
|
||||
"""A `Context` makes Gitea hunt for commit links, and on the API's render path that hunt
|
||||
dereferences a nil checker: the tracker's whole process segfaults. Every report carries a
|
||||
hex-shaped id, so the payload — not the text — is what has to stay safe (measured against
|
||||
1.26.4 / 1.27.1, fixed upstream in 1.27.3; a self-hoster's version is not ours to pick)."""
|
||||
owner = as_user(make_user())
|
||||
sid = _start(owner)["id"]
|
||||
text = "| tab | 5645b1c0ae01 |"
|
||||
assert owner.post(f"/api/debug-sessions/{sid}/preview", json={"text": text}).status_code == 200
|
||||
assert gitea_fake.markdown_payloads == [{"Text": text, "Mode": "gfm"}]
|
||||
|
||||
|
||||
def test_the_admin_connection_test_names_what_is_wrong(gitea_fake, db, make_user, as_user):
|
||||
admin = as_user(make_user(role="admin"))
|
||||
r = admin.post("/api/admin/config/test-issues")
|
||||
|
||||
@@ -180,7 +180,7 @@ def test_continue_watching_merges_both_sources_by_recency(db):
|
||||
)
|
||||
db.commit()
|
||||
|
||||
got = rails.continue_watching(db, user)
|
||||
got = rails.continue_watching(db, user, lang="en")
|
||||
assert [(c["source"], c["id"]) for c in got] == [("plex", "pm1"), ("youtube", "vmed")]
|
||||
assert got[0]["remaining_seconds"] == 100 * 60 - 600
|
||||
|
||||
@@ -208,7 +208,7 @@ def test_continue_watching_is_youtube_only_when_plex_is_off(db):
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
assert {c["source"] for c in rails.continue_watching(db, user)} == {"youtube"}
|
||||
assert {c["source"] for c in rails.continue_watching(db, user, lang="en")} == {"youtube"}
|
||||
|
||||
|
||||
def test_the_resume_rail_obeys_the_channel_blocklist(db):
|
||||
@@ -227,13 +227,13 @@ def test_the_resume_rail_obeys_the_channel_blocklist(db):
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
assert [c["id"] for c in rails.continue_watching(db, user)] == ["vmed"]
|
||||
assert [c["id"] for c in rails.continue_watching(db, user, lang="en")] == ["vmed"]
|
||||
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=ch.id))
|
||||
db.commit()
|
||||
assert rails.continue_watching(db, user) == []
|
||||
assert rails.continue_watching(db, user, lang="en") == []
|
||||
# And so the hero, whose resume tier draws from that rail, has nothing to offer either.
|
||||
assert hero.pick(db, user, "any", 0) is None
|
||||
assert hero.pick(db, user, "any", 0, lang="en") is None
|
||||
|
||||
|
||||
def test_a_null_timestamp_sorts_last(db):
|
||||
@@ -263,7 +263,7 @@ def test_a_null_timestamp_sorts_last(db):
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
got = rails.continue_watching(db, user)
|
||||
got = rails.continue_watching(db, user, lang="en")
|
||||
assert got[0]["id"] == "vshort"
|
||||
assert len(got) == 3
|
||||
|
||||
@@ -283,7 +283,7 @@ def test_hero_prefers_resume_and_labels_the_tier(db):
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
got = hero.pick(db, user, "any", seed=1)
|
||||
got = hero.pick(db, user, "any", seed=1, lang="en")
|
||||
assert got["tier"] == "resume"
|
||||
assert got["card"]["id"] == "vmed"
|
||||
|
||||
@@ -302,7 +302,7 @@ def test_hero_skips_a_resume_that_does_not_fit_the_budget(db):
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
got = hero.pick(db, user, "short", seed=1)
|
||||
got = hero.pick(db, user, "short", seed=1, lang="en")
|
||||
assert got["tier"] == "surprise"
|
||||
assert got["card"]["duration_seconds"] <= BANDS["short"][1]
|
||||
|
||||
@@ -312,7 +312,7 @@ def test_hero_falls_back_to_the_shelf_before_guessing(db):
|
||||
user, ch = _seed_yt(db)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
||||
db.commit()
|
||||
got = hero.pick(db, user, "medium", seed=1)
|
||||
got = hero.pick(db, user, "medium", seed=1, lang="en")
|
||||
assert got["tier"] == "comfort"
|
||||
assert got["card"]["id"] == "vmed"
|
||||
|
||||
@@ -332,7 +332,7 @@ def test_hero_resolves_a_show_to_the_episode_that_would_play(db):
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
got = hero.pick(db, user, "any", seed=1)
|
||||
got = hero.pick(db, user, "any", seed=1, lang="en")
|
||||
assert got["tier"] == "resume"
|
||||
assert got["card"]["kind"] == "episode"
|
||||
assert got["card"]["id"] == "ps1e2" # ep1 is finished → ep2 is what plays
|
||||
@@ -345,17 +345,19 @@ def test_hero_is_none_when_nothing_fits(db):
|
||||
user, ch = _seed_yt(db)
|
||||
db.execute(text("DELETE FROM videos WHERE id = 'vlong'"))
|
||||
db.commit()
|
||||
assert hero.pick(db, user, "long", seed=1) is None
|
||||
assert hero.pick(db, user, "long", seed=1, lang="en") is None
|
||||
|
||||
|
||||
def test_surprise_is_reproducible_for_a_seed(db):
|
||||
"""The reroll button depends on this: one seed drives both sources, so the same seed must give
|
||||
the same answer and a different seed must be free to give another."""
|
||||
user, _ = _seed_yt(db)
|
||||
a = rails.surprise(db, user, "any", seed=7, limit=3)
|
||||
b = rails.surprise(db, user, "any", seed=7, limit=3)
|
||||
a = rails.surprise(db, user, "any", seed=7, limit=3, lang="en")
|
||||
b = rails.surprise(db, user, "any", seed=7, limit=3, lang="en")
|
||||
assert [c["id"] for c in a] == [c["id"] for c in b]
|
||||
seeds = {tuple(c["id"] for c in rails.surprise(db, user, "any", s, 3)) for s in range(12)}
|
||||
seeds = {
|
||||
tuple(c["id"] for c in rails.surprise(db, user, "any", s, 3, lang="en")) for s in range(12)
|
||||
}
|
||||
assert len(seeds) > 1 # the seed actually reaches the ordering
|
||||
|
||||
|
||||
@@ -374,7 +376,9 @@ def test_surprise_never_offers_something_already_started(db):
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
assert "vlong" not in {c["id"] for c in rails.surprise(db, user, "long", seed=3, limit=5)}
|
||||
assert "vlong" not in {
|
||||
c["id"] for c in rails.surprise(db, user, "long", seed=3, limit=5, lang="en")
|
||||
}
|
||||
|
||||
|
||||
def test_the_seed_decides_which_source_leads(db):
|
||||
@@ -384,7 +388,9 @@ def test_the_seed_decides_which_source_leads(db):
|
||||
user, _ = _seed_yt(db)
|
||||
_seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
leaders = {rails.surprise(db, user, "any", s, limit=1)[0]["source"] for s in range(6)}
|
||||
leaders = {
|
||||
rails.surprise(db, user, "any", s, limit=1, lang="en")[0]["source"] for s in range(6)
|
||||
}
|
||||
assert leaders == {"youtube", "plex"}
|
||||
|
||||
|
||||
@@ -413,10 +419,10 @@ def test_the_reroll_must_be_able_to_move_past_the_resume_tier(db):
|
||||
)
|
||||
db.commit()
|
||||
|
||||
first = hero.pick(db, user, "any", seed=1)
|
||||
first = hero.pick(db, user, "any", seed=1, lang="en")
|
||||
assert first["card"]["id"] == "vmed"
|
||||
# Same seed, same everything — only the exclusion differs.
|
||||
second = hero.pick(db, user, "any", seed=1, exclude={"youtube:vmed"})
|
||||
second = hero.pick(db, user, "any", seed=1, exclude={"youtube:vmed"}, lang="en")
|
||||
assert second["card"]["id"] == "vshort"
|
||||
|
||||
|
||||
@@ -438,7 +444,7 @@ def test_the_reroll_rotates_which_tier_answers(db):
|
||||
)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vlong", position=0))
|
||||
db.commit()
|
||||
tiers = [hero.pick(db, user, "any", seed=1, start=n)["tier"] for n in range(4)]
|
||||
tiers = [hero.pick(db, user, "any", seed=1, start=n, lang="en")["tier"] for n in range(4)]
|
||||
assert tiers == ["resume", "comfort", "surprise", "resume"]
|
||||
|
||||
|
||||
@@ -447,7 +453,7 @@ def test_the_rotation_still_falls_through_an_empty_tier(db):
|
||||
Here there is no shelf at all, so the comfort-led press has to answer from somewhere else."""
|
||||
user, _ = _seed_yt(db)
|
||||
assert db.execute(select(func.count()).select_from(ComfortPin)).scalar_one() == 0
|
||||
got = hero.pick(db, user, "any", seed=1, start=1)
|
||||
got = hero.pick(db, user, "any", seed=1, start=1, lang="en")
|
||||
assert got is not None and got["tier"] != "comfort"
|
||||
|
||||
|
||||
@@ -468,10 +474,10 @@ def test_excluding_a_show_also_excludes_the_episode_it_resolves_to(db):
|
||||
)
|
||||
db.commit()
|
||||
|
||||
shown = hero.pick(db, user, "any", seed=1)
|
||||
shown = hero.pick(db, user, "any", seed=1, lang="en")
|
||||
assert shown["card"]["id"] == "ps1e1" # the show resolved to its in-progress episode
|
||||
# Excluding what the USER saw — the show — must move the tier on, not return the episode again.
|
||||
after = hero.pick(db, user, "any", seed=1, exclude={"plex:ps1"})
|
||||
after = hero.pick(db, user, "any", seed=1, exclude={"plex:ps1"}, lang="en")
|
||||
assert after is None or after["card"]["id"] != "ps1e1"
|
||||
|
||||
|
||||
@@ -518,7 +524,7 @@ def test_shelf_resolves_a_channel_to_its_newest_unwatched_video(db):
|
||||
user, ch = _seed_yt(db)
|
||||
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=0))
|
||||
db.commit()
|
||||
(card,) = pins.shelf(db, user)
|
||||
(card,) = pins.shelf(db, user, lang="en")
|
||||
assert card["kind"] == "channel"
|
||||
assert card["unwatched_count"] == 3
|
||||
assert card["next"]["id"] == "vshort" # newest
|
||||
@@ -538,7 +544,7 @@ def test_shelf_resolves_a_playlist_to_its_first_unwatched_item(db):
|
||||
db.add(VideoState(user_id=user.id, video_id="vshort", status="watched", watched_at=NOW))
|
||||
db.add(ComfortPin(user_id=user.id, playlist_id=pl.id, position=0))
|
||||
db.commit()
|
||||
(card,) = pins.shelf(db, user)
|
||||
(card,) = pins.shelf(db, user, lang="en")
|
||||
assert card["kind"] == "playlist"
|
||||
assert card["item_count"] == 2
|
||||
assert card["next"]["id"] == "vmed"
|
||||
@@ -585,7 +591,7 @@ def test_the_shelf_resolves_each_channel_and_playlist_on_its_own_terms(db):
|
||||
|
||||
got = [
|
||||
(c.get("unwatched_count", c.get("item_count")), (c["next"] or {}).get("id"))
|
||||
for c in pins.shelf(db, user)
|
||||
for c in pins.shelf(db, user, lang="en")
|
||||
]
|
||||
assert got == [(3, "vshort"), (1, "vother"), (1, "vlong"), (0, None)]
|
||||
|
||||
@@ -601,14 +607,14 @@ def test_the_shelf_costs_the_same_queries_for_one_pin_or_many(db):
|
||||
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=0))
|
||||
db.add(ComfortPin(user_id=user.id, playlist_id=lists[0].id, position=1))
|
||||
db.commit()
|
||||
few = _count_statements(db, lambda: pins.shelf(db, user))
|
||||
few = _count_statements(db, lambda: pins.shelf(db, user, lang="en"))
|
||||
|
||||
for i, c in enumerate(extra):
|
||||
db.add(ComfortPin(user_id=user.id, channel_id=c.id, position=10 + i))
|
||||
for i, p in enumerate(lists[1:]):
|
||||
db.add(ComfortPin(user_id=user.id, playlist_id=p.id, position=20 + i))
|
||||
db.commit()
|
||||
assert _count_statements(db, lambda: pins.shelf(db, user)) == few
|
||||
assert _count_statements(db, lambda: pins.shelf(db, user, lang="en")) == few
|
||||
|
||||
|
||||
def test_a_pinned_video_survives_leaving_the_feed_scope(db):
|
||||
@@ -619,7 +625,7 @@ def test_a_pinned_video_survives_leaving_the_feed_scope(db):
|
||||
db.commit()
|
||||
db.execute(delete(Subscription))
|
||||
db.commit()
|
||||
(card,) = pins.shelf(db, user)
|
||||
(card,) = pins.shelf(db, user, lang="en")
|
||||
assert card["id"] == "vmed"
|
||||
assert card["next"]["id"] == "vmed"
|
||||
|
||||
@@ -633,7 +639,9 @@ def test_the_plex_surprise_obeys_the_time_budget(db):
|
||||
_seed_plex(db, user)
|
||||
_enable_plex(db)
|
||||
drawn = lambda budget: { # noqa: E731
|
||||
(c["source"], c["id"]) for s in range(6) for c in rails.surprise(db, user, budget, s, 5)
|
||||
(c["source"], c["id"])
|
||||
for s in range(6)
|
||||
for c in rails.surprise(db, user, budget, s, 5, lang="en")
|
||||
}
|
||||
assert ("plex", "pm1") not in drawn("short")
|
||||
# …and it IS offered where it fits, so this cannot pass by the rail being empty.
|
||||
@@ -658,7 +666,9 @@ def test_the_plex_surprise_never_offers_something_already_started(db):
|
||||
)
|
||||
db.commit()
|
||||
drawn = {
|
||||
(c["source"], c["id"]) for s in range(6) for c in rails.surprise(db, user, "long", s, 5)
|
||||
(c["source"], c["id"])
|
||||
for s in range(6)
|
||||
for c in rails.surprise(db, user, "long", s, 5, lang="en")
|
||||
}
|
||||
assert ("plex", "pm1") not in drawn
|
||||
|
||||
@@ -682,7 +692,7 @@ def test_a_new_pin_lands_at_the_end_even_after_a_hole(db):
|
||||
|
||||
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=pins.next_position(db, user)))
|
||||
db.commit()
|
||||
assert [c["kind"] for c in pins.shelf(db, user)] == ["video", "channel"]
|
||||
assert [c["kind"] for c in pins.shelf(db, user, lang="en")] == ["video", "channel"]
|
||||
positions = (
|
||||
db.execute(select(ComfortPin.position).order_by(ComfortPin.position)).scalars().all()
|
||||
)
|
||||
@@ -695,7 +705,7 @@ def test_a_self_playing_pin_does_not_reference_itself(db):
|
||||
user, _ = _seed_yt(db)
|
||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
||||
db.commit()
|
||||
(card,) = pins.shelf(db, user)
|
||||
(card,) = pins.shelf(db, user, lang="en")
|
||||
assert card["next"] is not card
|
||||
assert "next" not in card["next"]
|
||||
|
||||
@@ -718,7 +728,7 @@ def test_the_resume_rail_says_what_would_actually_play(db):
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
rail = rails.continue_watching(db, user)
|
||||
rail = rails.continue_watching(db, user, lang="en")
|
||||
tile = next(c for c in rail if c["kind"] == "show")
|
||||
assert tile["id"] == "ps1" # still the show
|
||||
assert tile["next"]["kind"] == "episode"
|
||||
@@ -762,7 +772,7 @@ def test_only_a_show_row_pays_for_a_resolution(db):
|
||||
)
|
||||
db.commit()
|
||||
|
||||
rail = rails.continue_watching(db, user)
|
||||
rail = rails.continue_watching(db, user, lang="en")
|
||||
# A video, a movie and a show — one row of each kind that can appear here, so the assertion below
|
||||
# actually exercises both sides of the equality.
|
||||
assert {c["kind"] for c in rail} == {"video", "movie", "show"}
|
||||
@@ -778,7 +788,7 @@ def test_next_episode_takes_an_episode_with_no_state_row_at_all(db):
|
||||
_, show, ep1 = _seed_plex(db, user)
|
||||
db.add(PlexState(user_id=user.id, item_id=ep1.id, status="watched", watched_at=NOW))
|
||||
db.commit() # ep2 has NO PlexState row
|
||||
assert queries.next_episode(db, user.id, show.id)["id"] == "ps1e2"
|
||||
assert queries.next_episode(db, user.id, show.id, lang="en")["id"] == "ps1e2"
|
||||
|
||||
|
||||
def test_a_finished_show_starts_again_from_the_top(db):
|
||||
@@ -793,7 +803,7 @@ def test_a_finished_show_starts_again_from_the_top(db):
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
assert queries.next_episode(db, user.id, show.id)["id"] == "ps1e1"
|
||||
assert queries.next_episode(db, user.id, show.id, lang="en")["id"] == "ps1e1"
|
||||
|
||||
|
||||
def test_next_episode_agrees_with_the_show_page_when_two_are_in_progress(db):
|
||||
@@ -825,12 +835,16 @@ def test_next_episode_agrees_with_the_show_page_when_two_are_in_progress(db):
|
||||
)
|
||||
db.commit()
|
||||
|
||||
got = queries.next_episode(db, user.id, show.id)
|
||||
got = queries.next_episode(db, user.id, show.id, lang="en")
|
||||
assert got["id"] == "ps1e2"
|
||||
# Asserted against the other implementation rather than against a literal, so the two cannot
|
||||
# drift apart again without this failing.
|
||||
ordered = [
|
||||
leaf_card(ep, db.execute(select(PlexState).where(PlexState.item_id == ep.id)).scalar_one())
|
||||
leaf_card(
|
||||
ep,
|
||||
db.execute(select(PlexState).where(PlexState.item_id == ep.id)).scalar_one(),
|
||||
lang="en",
|
||||
)
|
||||
for ep in (ep1, ep2)
|
||||
]
|
||||
assert rollup.rollup(ordered)["resume"]["id"] == got["id"]
|
||||
@@ -848,7 +862,7 @@ def test_next_episode_prefers_the_one_in_progress(db):
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
assert queries.next_episode(db, user.id, show.id)["id"] == "ps1e2"
|
||||
assert queries.next_episode(db, user.id, show.id, lang="en")["id"] == "ps1e2"
|
||||
|
||||
|
||||
# ── HTTP surface ──────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""E30: the "shared with you" notice carries the download's title in its data, so the client can name
|
||||
the notice in the reader's language. Its stored title is an English sentence ("Cats was shared with
|
||||
you") — the only place the title lived, so a Hungarian reader got the English. Needs the DB."""
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.models import DownloadJob, MediaAsset, Notification
|
||||
|
||||
|
||||
def test_a_share_notice_carries_the_title_and_the_sharer(db, make_user, as_user):
|
||||
owner = make_user("owner@example.com")
|
||||
friend = make_user("friend@example.com")
|
||||
asset = MediaAsset(
|
||||
source_kind="url", source_ref="share-e30", format_sig="sig", status="ready", ref_count=1
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
job = DownloadJob(
|
||||
user_id=owner.id,
|
||||
asset_id=asset.id,
|
||||
source_kind="url",
|
||||
source_ref="share-e30",
|
||||
status="done",
|
||||
queue_pos=1,
|
||||
profile_snapshot={},
|
||||
display_name="Cats",
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
|
||||
r = as_user(owner).post(f"/api/downloads/{job.id}/share", json={"email": "friend@example.com"})
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
notice = db.execute(select(Notification).where(Notification.user_id == friend.id)).scalar_one()
|
||||
assert notice.type == "download_shared"
|
||||
assert notice.data == {
|
||||
"kind": "download_shared",
|
||||
"from": "owner@example.com",
|
||||
"job_id": job.id,
|
||||
"title": "Cats",
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
"""The Stats page's "Full history: done / requested" ratio counts the SAME channels on both sides — the
|
||||
ones this user asked full history for. The numerator used to count every subscribed channel whose
|
||||
back-catalog was complete (another user had asked, or the channel was short), so the page read
|
||||
"342/329". "Channels without full history yet" still counts every subscribed channel. Needs the DB."""
|
||||
|
||||
from app.models import Channel, Subscription
|
||||
|
||||
|
||||
def test_full_history_ratio_counts_only_the_requested_channels(db, make_user, as_user):
|
||||
user = make_user("deep@example.com")
|
||||
db.add_all(
|
||||
[
|
||||
Channel(id="UCreqdone", title="Requested, done", backfill_done=True),
|
||||
Channel(id="UCreqopen", title="Requested, pending", backfill_done=False),
|
||||
Channel(id="UCdone", title="Not requested, done anyway", backfill_done=True),
|
||||
Channel(id="UCopen", title="Not requested, pending", backfill_done=False),
|
||||
]
|
||||
)
|
||||
db.flush()
|
||||
db.add_all(
|
||||
[
|
||||
Subscription(user_id=user.id, channel_id="UCreqdone", deep_requested=True),
|
||||
Subscription(user_id=user.id, channel_id="UCreqopen", deep_requested=True),
|
||||
Subscription(user_id=user.id, channel_id="UCdone"),
|
||||
Subscription(user_id=user.id, channel_id="UCopen"),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
body = as_user(user).get("/api/sync/my-status").json()
|
||||
assert body["channels_deep_requested"] == 2
|
||||
assert body["channels_deep_done"] == 1 # never more than it is out of
|
||||
assert body["channels_deep_pending"] == 2 # subscribed channels without full history yet
|
||||
@@ -0,0 +1,105 @@
|
||||
"""E28 S3e2 — adding a video AT a position: the list page's "Undo" of a removal.
|
||||
|
||||
The row has to come back where it was in ONE request. The two-call shape (append, then send the
|
||||
whole order) leaves it at the end of the list whenever the second call fails, and the reorder would
|
||||
also overwrite any change another tab made in between. Positions can have gaps after removals, so
|
||||
the position is an index over the ORDERED rows, and the list is renumbered around it.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.models import Channel, Playlist, PlaylistItem, Video
|
||||
|
||||
|
||||
def _seed(db, user, ids, *, positions=None, **kw):
|
||||
db.add(Channel(id="UCtest", title="Test"))
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in [*ids, "vX"]:
|
||||
db.add(Video(id=v, channel_id="UCtest", published_at=now))
|
||||
pl = Playlist(user_id=user.id, name="L", **kw)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for i, vid in enumerate(ids):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=(positions or range(99))[i]))
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
def _ids(db, pl_id):
|
||||
db.expire_all()
|
||||
return [
|
||||
r.video_id
|
||||
for r in db.execute(
|
||||
PlaylistItem.__table__.select()
|
||||
.where(PlaylistItem.playlist_id == pl_id)
|
||||
.order_by(PlaylistItem.position)
|
||||
).all()
|
||||
]
|
||||
|
||||
|
||||
def _add(client, pl_id, body):
|
||||
return client.post(f"/api/playlists/{pl_id}/items", json=body)
|
||||
|
||||
|
||||
def test_a_position_puts_the_video_back_in_its_place(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"])
|
||||
r = _add(as_user(user), pl.id, {"video_id": "vX", "position": 1})
|
||||
assert r.status_code == 200
|
||||
assert _ids(db, pl.id) == ["a", "vX", "b", "c"]
|
||||
|
||||
|
||||
def test_the_position_counts_rows_not_stored_numbers(db, make_user, as_user):
|
||||
# A removal leaves a gap (0, 5, 9): index 2 is before the THIRD row, not "position 2".
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"], positions=[0, 5, 9])
|
||||
_add(as_user(user), pl.id, {"video_id": "vX", "position": 2})
|
||||
assert _ids(db, pl.id) == ["a", "b", "vX", "c"]
|
||||
|
||||
|
||||
def test_a_position_past_the_end_appends(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b"])
|
||||
_add(as_user(user), pl.id, {"video_id": "vX", "position": 40})
|
||||
assert _ids(db, pl.id) == ["a", "b", "vX"]
|
||||
|
||||
|
||||
def test_no_position_still_appends(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b"])
|
||||
_add(as_user(user), pl.id, {"video_id": "vX"})
|
||||
assert _ids(db, pl.id) == ["a", "b", "vX"]
|
||||
|
||||
|
||||
def test_a_video_already_in_the_list_does_not_move(db, make_user, as_user):
|
||||
# The add is idempotent; a position must not turn it into a reorder.
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"])
|
||||
_add(as_user(user), pl.id, {"video_id": "c", "position": 0})
|
||||
assert _ids(db, pl.id) == ["a", "b", "c"]
|
||||
|
||||
|
||||
def test_a_negative_position_is_refused(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a"])
|
||||
r = _add(as_user(user), pl.id, {"video_id": "vX", "position": -1})
|
||||
assert r.status_code == 422
|
||||
assert _ids(db, pl.id) == ["a"]
|
||||
|
||||
|
||||
def test_putting_a_removed_video_back_leaves_a_linked_list_clean(db, make_user, as_user):
|
||||
# Remove + undo is no change at all: a YouTube-linked list must not stay "unsynced" after it.
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"], yt_playlist_id="PLx")
|
||||
client = as_user(user)
|
||||
from app.sync.playlists import current_fingerprint
|
||||
|
||||
pl.synced_fingerprint = current_fingerprint(db, pl)
|
||||
pl.dirty = False
|
||||
db.commit()
|
||||
client.delete(f"/api/playlists/{pl.id}/items/b")
|
||||
db.expire_all()
|
||||
assert db.get(Playlist, pl.id).dirty is True
|
||||
_add(client, pl.id, {"video_id": "b", "position": 1})
|
||||
db.expire_all()
|
||||
assert db.get(Playlist, pl.id).dirty is False
|
||||
@@ -13,6 +13,9 @@ Two properties are load-bearing and neither is visible from the happy path alone
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.exc import OperationalError
|
||||
|
||||
from app.models import Channel, Playlist, PlaylistItem, Video
|
||||
|
||||
|
||||
@@ -269,12 +272,12 @@ def test_a_target_that_already_had_it_is_not_recently_changed(db, make_user, as_
|
||||
assert db.get(Playlist, dst.id).updated_at == old
|
||||
|
||||
|
||||
def test_a_concurrent_add_to_the_target_lands_as_the_already_present_move(
|
||||
db, make_user, as_user, monkeypatch
|
||||
):
|
||||
"""The already-in-target check is a read, not a lock: another tab's add (or the scheduler's
|
||||
YouTube merge) can commit the same video to the target in between, and the re-parent then
|
||||
hits uq_playlist_video. That is the idempotent outcome arriving late — not a raw 500."""
|
||||
def test_no_add_can_land_in_the_target_while_a_move_holds_it(db, make_user):
|
||||
"""The already-in-target check is a read; it holds because the move locks both lists first and
|
||||
an insert into the target — another tab's add, a YouTube merge — needs the target row for its
|
||||
foreign key, so it waits for the move's commit instead of hitting uq_playlist_video under it."""
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.routes import playlists as plroutes
|
||||
|
||||
@@ -282,16 +285,10 @@ def test_a_concurrent_add_to_the_target_lands_as_the_already_present_move(
|
||||
_seed_videos(db, ["v1"])
|
||||
src = _playlist(db, user, "Source", ["v1"])
|
||||
dst = _playlist(db, user, "Target", [])
|
||||
real = plroutes._next_item_position
|
||||
|
||||
def add_first(session, playlist_id):
|
||||
with SessionLocal() as other: # the other tab, committing between check and re-parent
|
||||
other.add(PlaylistItem(playlist_id=dst.id, video_id="v1", position=0))
|
||||
other.commit()
|
||||
return real(session, playlist_id)
|
||||
|
||||
monkeypatch.setattr(plroutes, "_next_item_position", add_first)
|
||||
assert _move(as_user(user), src.id, "v1", dst.id).status_code == 200
|
||||
db.expire_all()
|
||||
assert _ids(db, src.id) == []
|
||||
assert _ids(db, dst.id) == ["v1"]
|
||||
plroutes.lock_playlists(db, src.id, dst.id) # what the move holds until its commit
|
||||
with SessionLocal() as other:
|
||||
other.execute(text("SET LOCAL lock_timeout = '300ms'"))
|
||||
other.add(PlaylistItem(playlist_id=dst.id, video_id="v1", position=0))
|
||||
with pytest.raises(OperationalError, match="lock timeout"):
|
||||
other.flush()
|
||||
db.rollback()
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
"""E28 review — every request that reads a playlist's item positions and writes new ones locks the
|
||||
playlist row first.
|
||||
|
||||
Without it two adds (two tabs; the rail and the page) both read the same `max(position)` and land
|
||||
on one position, and a reorder racing an add collides with it — the list's order between the two
|
||||
rows is then whatever the planner returns. A true two-connection race is not reproducible on
|
||||
demand, so the test holds the mechanism: the parent row is taken `FOR UPDATE` by each such route.
|
||||
A Plex reorder that omits a row (added in another tab meanwhile) must not collide with it either.
|
||||
|
||||
Round 2: the lock also comes FIRST — before the request reads or writes a single item row. A remove
|
||||
that deleted an item and then touched the playlist beside a reorder that went the other way was a
|
||||
deadlock (a 500 for one of them), and an add that checked "already there?" before the lock let two
|
||||
adds of one video both insert (a 500 on the unique key).
|
||||
"""
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import event
|
||||
|
||||
from app.models import (
|
||||
Channel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
PlexPlaylist,
|
||||
PlexPlaylistItem,
|
||||
Video,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def locks(db):
|
||||
"""The tables each request took a row lock on (`SELECT … FROM <t> … FOR UPDATE`)."""
|
||||
seen: list[str] = []
|
||||
|
||||
def _record(conn, cursor, statement, *_):
|
||||
if "FOR UPDATE" in statement:
|
||||
seen.extend(re.findall(r"FROM\s+(\w+)", statement)[:1])
|
||||
|
||||
engine = db.get_bind()
|
||||
event.listen(engine, "before_cursor_execute", _record)
|
||||
yield seen
|
||||
event.remove(engine, "before_cursor_execute", _record)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def statements(db):
|
||||
"""Every statement a request ran, in order."""
|
||||
seen: list[str] = []
|
||||
|
||||
def _record(conn, cursor, statement, *_):
|
||||
seen.append(" ".join(statement.split()))
|
||||
|
||||
engine = db.get_bind()
|
||||
event.listen(engine, "before_cursor_execute", _record)
|
||||
yield seen
|
||||
event.remove(engine, "before_cursor_execute", _record)
|
||||
|
||||
|
||||
def _locked_before_items(seen: list[str], parent: str, items: str) -> str | None:
|
||||
"""The first statement touching `items` must come after a `FOR UPDATE` of `parent`; returns
|
||||
that lock statement (None when it came too late or never)."""
|
||||
lock = None
|
||||
for st in seen:
|
||||
if lock is None and "FOR UPDATE" in st and re.search(rf"FROM {parent}\b", st):
|
||||
lock = st
|
||||
if re.search(rf"\b{items}\b", st):
|
||||
return lock
|
||||
return lock
|
||||
|
||||
|
||||
def _yt(db, user):
|
||||
db.add(Channel(id="UCtest", title="Test"))
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in ("v1", "v2", "v3"):
|
||||
db.add(Video(id=v, channel_id="UCtest", published_at=now))
|
||||
pl = Playlist(user_id=user.id, name="L")
|
||||
other = Playlist(user_id=user.id, name="M")
|
||||
db.add_all([pl, other])
|
||||
db.flush()
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id="v1", position=0))
|
||||
db.commit()
|
||||
return pl.id, other.id
|
||||
|
||||
|
||||
def _plex(db, user):
|
||||
lib = PlexLibrary(plex_key="p1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
items = [PlexItem(rating_key=rk, library_id=lib.id, kind="movie", title=rk) for rk in "abcd"]
|
||||
db.add_all(items)
|
||||
pl = PlexPlaylist(user_id=user.id, title="L")
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for i, it in enumerate(items[:3]):
|
||||
db.add(PlexPlaylistItem(playlist_id=pl.id, item_id=it.id, position=i))
|
||||
db.commit()
|
||||
return pl.id
|
||||
|
||||
|
||||
YT = Playlist.__table__.name
|
||||
PLEX = PlexPlaylist.__table__.name
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call",
|
||||
[
|
||||
lambda c, pid, other: c.post(f"/api/playlists/{pid}/items", json={"video_id": "v2"}),
|
||||
lambda c, pid, other: c.post(
|
||||
f"/api/playlists/{pid}/items", json={"video_id": "v2", "position": 0}
|
||||
),
|
||||
lambda c, pid, other: c.post("/api/playlists/watch-later", json={"video_id": "v2"}),
|
||||
lambda c, pid, other: c.post(
|
||||
f"/api/playlists/{pid}/items/v1/move", json={"target_playlist_id": other}
|
||||
),
|
||||
lambda c, pid, other: c.put(f"/api/playlists/{pid}/order", json={"video_ids": ["v1"]}),
|
||||
],
|
||||
ids=["add", "add-at", "watch-later", "move", "reorder"],
|
||||
)
|
||||
def test_a_youtube_playlist_write_locks_the_playlist(db, make_user, as_user, locks, call):
|
||||
user = make_user()
|
||||
pid, other = _yt(db, user)
|
||||
r = call(as_user(user), pid, other)
|
||||
assert r.status_code == 200, r.text
|
||||
assert YT in locks
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call",
|
||||
[
|
||||
lambda c, pid: c.post(f"/api/plex/playlists/{pid}/items", json={"item_rating_key": "d"}),
|
||||
lambda c, pid: c.post(
|
||||
f"/api/plex/playlists/{pid}/items/bulk", json={"item_rating_keys": ["d"]}
|
||||
),
|
||||
lambda c, pid: c.post(
|
||||
f"/api/plex/playlists/{pid}/items/bulk",
|
||||
json={"item_rating_keys": ["d"], "positions": [0]},
|
||||
),
|
||||
lambda c, pid: c.put(
|
||||
f"/api/plex/playlists/{pid}/order", json={"item_rating_keys": ["c", "b", "a"]}
|
||||
),
|
||||
],
|
||||
ids=["add", "bulk", "bulk-at", "reorder"],
|
||||
)
|
||||
def test_a_plex_playlist_write_locks_the_playlist(db, make_user, as_user, locks, call):
|
||||
user = make_user()
|
||||
pid = _plex(db, user)
|
||||
r = call(as_user(user), pid)
|
||||
assert r.status_code == 200, r.text
|
||||
assert PLEX in locks
|
||||
|
||||
|
||||
def test_a_plex_reorder_that_omits_a_row_does_not_collide_with_it(db, make_user, as_user):
|
||||
"""The row "a" was added in another tab after this page loaded its order ["c", "b"]: it keeps
|
||||
its place after the given rows instead of sharing position 0 with "c"."""
|
||||
user = make_user()
|
||||
pid = _plex(db, user)
|
||||
r = as_user(user).put(f"/api/plex/playlists/{pid}/order", json={"item_rating_keys": ["c", "b"]})
|
||||
assert r.status_code == 200
|
||||
db.expire_all()
|
||||
rows = db.query(PlexPlaylistItem).filter_by(playlist_id=pid).all()
|
||||
by_id = {it.id: it.rating_key for it in db.query(PlexItem).all()}
|
||||
got = sorted((r.position, by_id[r.item_id]) for r in rows)
|
||||
assert got == [(0, "c"), (1, "b"), (2, "a")]
|
||||
|
||||
|
||||
YT_ITEMS = PlaylistItem.__table__.name
|
||||
PLEX_ITEMS = PlexPlaylistItem.__table__.name
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call",
|
||||
[
|
||||
lambda c, pid, other: c.post(f"/api/playlists/{pid}/items", json={"video_id": "v1"}),
|
||||
lambda c, pid, other: c.post(
|
||||
f"/api/playlists/{pid}/items", json={"video_id": "v2", "position": 0}
|
||||
),
|
||||
lambda c, pid, other: c.delete(f"/api/playlists/{pid}/items/v1"),
|
||||
lambda c, pid, other: c.post(
|
||||
f"/api/playlists/{pid}/items/v1/move", json={"target_playlist_id": other}
|
||||
),
|
||||
lambda c, pid, other: c.put(f"/api/playlists/{pid}/order", json={"video_ids": ["v1"]}),
|
||||
# the ORM cascade deletes the items before the list row
|
||||
lambda c, pid, other: c.delete(f"/api/playlists/{pid}"),
|
||||
],
|
||||
ids=["add-present", "add-at", "remove", "move", "reorder", "delete"],
|
||||
)
|
||||
def test_a_youtube_item_write_locks_the_playlist_before_any_item(
|
||||
db, make_user, as_user, statements, call
|
||||
):
|
||||
user = make_user()
|
||||
pid, other = _yt(db, user)
|
||||
c = as_user(user)
|
||||
statements.clear()
|
||||
r = call(c, pid, other)
|
||||
assert r.status_code == 200, r.text
|
||||
assert _locked_before_items(statements, YT, YT_ITEMS), statements
|
||||
|
||||
|
||||
def test_a_move_locks_both_lists_in_one_ordered_statement(db, make_user, as_user, statements):
|
||||
"""Two moves A→B and B→A each locking "its" list first would lock crosswise."""
|
||||
user = make_user()
|
||||
pid, other = _yt(db, user)
|
||||
statements.clear()
|
||||
r = as_user(user).post(
|
||||
f"/api/playlists/{pid}/items/v1/move", json={"target_playlist_id": other}
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
lock = _locked_before_items(statements, YT, YT_ITEMS)
|
||||
assert lock and "ORDER BY" in lock, lock
|
||||
# BOTH lists in that one statement: one bound id per list in its IN (…) (a single-id lock
|
||||
# renders the same IN and ORDER BY)
|
||||
listed = re.search(r" IN \((.*?)\) ORDER BY", lock)
|
||||
assert listed and len(re.findall(r"%\(\w+\)s", listed.group(1))) == 2, lock
|
||||
|
||||
|
||||
def test_watch_later_add_and_remove_lock_it_before_any_item(db, make_user, as_user, statements):
|
||||
user = make_user()
|
||||
_yt(db, user)
|
||||
c = as_user(user)
|
||||
assert c.post("/api/playlists/watch-later", json={"video_id": "v2"}).status_code == 200
|
||||
statements.clear()
|
||||
assert c.post("/api/playlists/watch-later", json={"video_id": "v2"}).status_code == 200
|
||||
assert _locked_before_items(statements, YT, YT_ITEMS), statements
|
||||
statements.clear()
|
||||
assert c.delete("/api/playlists/watch-later/v2").status_code == 200
|
||||
assert _locked_before_items(statements, YT, YT_ITEMS), statements
|
||||
|
||||
|
||||
def test_a_rename_locks_the_playlist(db, make_user, as_user, locks):
|
||||
"""A rename moves `dirty`, which the scheduled mirror refresh re-reads under this lock."""
|
||||
user = make_user()
|
||||
pid, _ = _yt(db, user)
|
||||
r = as_user(user).patch(f"/api/playlists/{pid}", json={"name": "New"})
|
||||
assert r.status_code == 200, r.text
|
||||
assert YT in locks
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"call",
|
||||
[
|
||||
lambda c, pid: c.delete(f"/api/plex/playlists/{pid}/items/a"),
|
||||
lambda c, pid: c.post(
|
||||
f"/api/plex/playlists/{pid}/items/remove-bulk", json={"item_rating_keys": ["a", "b"]}
|
||||
),
|
||||
lambda c, pid: c.post(f"/api/plex/playlists/{pid}/items", json={"item_rating_key": "a"}),
|
||||
],
|
||||
ids=["remove", "remove-bulk", "add-present"],
|
||||
)
|
||||
def test_a_plex_item_write_locks_the_playlist_before_any_item(
|
||||
db, make_user, as_user, statements, call
|
||||
):
|
||||
user = make_user()
|
||||
pid = _plex(db, user)
|
||||
c = as_user(user)
|
||||
statements.clear()
|
||||
r = call(c, pid)
|
||||
assert r.status_code == 200, r.text
|
||||
assert _locked_before_items(statements, PLEX, PLEX_ITEMS), statements
|
||||
@@ -0,0 +1,210 @@
|
||||
"""E28 review C1 — one YouTube sync of a playlist at a time, whoever starts it.
|
||||
|
||||
A push reads YouTube and then writes both sides for seconds, committing step by step, so the
|
||||
client's in-flight flag (one page's menus) and a row lock (gone at the first commit) both leave two
|
||||
tabs, or a push and the scheduled mirror refresh, free to interleave. The claim is an advisory lock
|
||||
on a connection of its own; the test holds it from outside, the way a second request would.
|
||||
"""
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.auth import require_write_scope
|
||||
from app.db import SessionLocal
|
||||
from app.models import Playlist
|
||||
from app.routes import playlists as plroutes
|
||||
from app.sync import playlists as plsync
|
||||
|
||||
|
||||
def _linked(db, user, *, source="local"):
|
||||
pl = Playlist(user_id=user.id, name="L", source=source, yt_playlist_id="PL123")
|
||||
db.add(pl)
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
class FakeYT:
|
||||
"""YouTube as far as these routes read it: two playlists, none with items."""
|
||||
|
||||
def __init__(self, *_):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
return False
|
||||
|
||||
def iter_my_playlists(self):
|
||||
return [{"id": "PL123", "title": "Renamed"}, {"id": "PL999", "title": "New"}]
|
||||
|
||||
def iter_my_playlist_video_ids(self, _ytid):
|
||||
return []
|
||||
|
||||
def iter_playlist_items_with_ids(self, _ytid):
|
||||
return []
|
||||
|
||||
def get_playlist_snippet(self, _ytid):
|
||||
return {}
|
||||
|
||||
|
||||
def test_push_and_revert_refuse_while_the_list_is_claimed(
|
||||
db, as_user, make_user, override_dep, monkeypatch
|
||||
):
|
||||
user = make_user()
|
||||
pl = _linked(db, user)
|
||||
override_dep(require_write_scope, lambda: user)
|
||||
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
||||
ran: list[str] = []
|
||||
monkeypatch.setattr(plroutes, "YouTubeClient", FakeYT)
|
||||
monkeypatch.setattr(plroutes, "plan_merge", lambda d, u, p, live: {"units_estimate": 0})
|
||||
monkeypatch.setattr(plroutes, "merge_playlist", lambda d, u, p, live: ran.append("push") or {})
|
||||
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: ran.append("revert") or {})
|
||||
c = as_user(user)
|
||||
|
||||
with plsync.sync_claim(db, pl.id) as free:
|
||||
assert free
|
||||
for path in ("push", "revert-youtube"):
|
||||
r = c.post(f"/api/playlists/{pl.id}/{path}")
|
||||
assert r.status_code == 409
|
||||
assert r.json()["detail"] == {"code": "playlist", "reason": "sync_busy"}
|
||||
# a delete too: a push mid-way would plan the deleted list as emptied (E28 round 4)
|
||||
r = c.delete(f"/api/playlists/{pl.id}")
|
||||
assert r.status_code == 409
|
||||
assert r.json()["detail"] == {"code": "playlist", "reason": "sync_busy"}
|
||||
assert ran == []
|
||||
assert db.get(Playlist, pl.id) is not None
|
||||
|
||||
# Released on the way out: the same list syncs again, and a claim is per list.
|
||||
assert c.post(f"/api/playlists/{pl.id}/push").status_code == 200
|
||||
assert c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code == 200
|
||||
assert ran == ["push", "revert"]
|
||||
|
||||
|
||||
def test_a_claim_is_released_when_its_block_raises(db, make_user):
|
||||
pl = _linked(db, make_user())
|
||||
try:
|
||||
with plsync.sync_claim(db, pl.id) as free:
|
||||
assert free
|
||||
raise RuntimeError("boom")
|
||||
except RuntimeError:
|
||||
pass
|
||||
# Asked of pg_locks, not by claiming again: a lock leaked on a pooled connection would be granted
|
||||
# again to that same connection (advisory locks are re-entrant per session).
|
||||
held = db.scalar(
|
||||
text("SELECT count(*) FROM pg_locks WHERE locktype = 'advisory' AND objid = :id"),
|
||||
{"id": pl.id},
|
||||
)
|
||||
assert held == 0
|
||||
|
||||
|
||||
def test_a_claim_draws_no_connection_from_the_app_pool(db, make_user):
|
||||
"""A claim is held for a whole push beside the request's pooled session; drawn from the pool
|
||||
too, a burst of pushes ran it dry and stalled every other request for pool_timeout."""
|
||||
from app.db import engine
|
||||
|
||||
pl = _linked(db, make_user())
|
||||
before = engine.pool.checkedout()
|
||||
with plsync.sync_claim(db, pl.id) as free:
|
||||
assert free
|
||||
assert engine.pool.checkedout() == before
|
||||
|
||||
|
||||
def test_the_scheduled_refresh_skips_a_mirror_being_synced(db, make_user, monkeypatch):
|
||||
user = make_user()
|
||||
busy = _linked(db, user, source="youtube")
|
||||
replaced: list[int] = []
|
||||
|
||||
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plsync, "YouTubeClient", FakeYT)
|
||||
monkeypatch.setattr(
|
||||
plsync, "_replace_items_from_youtube", lambda d, p, ids: replaced.append(p.id)
|
||||
)
|
||||
|
||||
with plsync.sync_claim(db, busy.id):
|
||||
out = plsync.sync_user_playlists(db, user)
|
||||
|
||||
assert out == {"synced": 1}
|
||||
assert busy.id not in replaced and len(replaced) == 1
|
||||
db.refresh(busy)
|
||||
assert busy.name == "L"
|
||||
|
||||
|
||||
def test_refused_reverts_do_not_spend_the_pull_cooldown(
|
||||
db, as_user, make_user, override_dep, monkeypatch
|
||||
):
|
||||
"""Round 2: the cooldown (three pulls per ten minutes) is checked after the claim, so clicking
|
||||
Revert while a push runs is refused as busy without using up a pull."""
|
||||
user = make_user()
|
||||
pl = _linked(db, user)
|
||||
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: {})
|
||||
c = as_user(user)
|
||||
with plsync.sync_claim(db, pl.id):
|
||||
for _ in range(4):
|
||||
assert c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code == 409
|
||||
assert c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code == 200
|
||||
|
||||
|
||||
def test_a_bulk_pull_runs_once_per_user_at_a_time(db, as_user, make_user, monkeypatch):
|
||||
"""Two bulk pulls at once both saw a YouTube list unmirrored and each created its mirror."""
|
||||
user = make_user()
|
||||
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plsync, "YouTubeClient", FakeYT)
|
||||
monkeypatch.setattr(plsync, "_replace_items_from_youtube", lambda d, p, ids: None)
|
||||
c = as_user(user)
|
||||
with plsync.user_sync_claim(db, user.id):
|
||||
assert plsync.sync_user_playlists(db, user) == {"synced": 0, "reason": "busy"}
|
||||
for _ in range(4):
|
||||
r = c.post("/api/playlists/sync-youtube")
|
||||
assert r.status_code == 409
|
||||
assert r.json()["detail"] == {"code": "playlist", "reason": "sync_all_busy"}
|
||||
assert c.post("/api/playlists/sync-youtube").json() == {"ok": True, "synced": 2}
|
||||
|
||||
|
||||
def test_a_gone_mirror_is_deleted_even_when_no_list_refreshes(db, make_user, monkeypatch):
|
||||
"""Every remaining mirror has unpushed edits, so nothing below the deletion commits."""
|
||||
user = make_user()
|
||||
gone = Playlist(user_id=user.id, name="Gone", source="youtube", yt_playlist_id="PLgone")
|
||||
kept = Playlist(user_id=user.id, name="K", source="youtube", yt_playlist_id="PL123", dirty=True)
|
||||
other = Playlist(
|
||||
user_id=user.id, name="O", source="youtube", yt_playlist_id="PL999", dirty=True
|
||||
)
|
||||
db.add_all([gone, kept, other])
|
||||
db.commit()
|
||||
gone_id = gone.id
|
||||
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plsync, "YouTubeClient", FakeYT)
|
||||
|
||||
assert plsync.sync_user_playlists(db, user) == {"synced": 0}
|
||||
db.rollback() # what the request's session close does to an uncommitted deletion
|
||||
assert db.get(Playlist, gone_id) is None
|
||||
|
||||
|
||||
def test_a_mirror_edited_during_the_pull_is_not_replaced(db, make_user, monkeypatch):
|
||||
"""The loop's `dirty` is a snapshot from before the YouTube calls; a remove committed by another
|
||||
request meanwhile must win over YouTube's order."""
|
||||
user = make_user()
|
||||
pl = _linked(db, user, source="youtube")
|
||||
replaced: list[int] = []
|
||||
|
||||
class EditedMeanwhile(FakeYT):
|
||||
def iter_my_playlists(self):
|
||||
return [{"id": "PL123", "title": "Renamed"}]
|
||||
|
||||
def iter_my_playlist_video_ids(self, _ytid):
|
||||
with SessionLocal() as other:
|
||||
other.get(Playlist, pl.id).dirty = True
|
||||
other.commit()
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(plsync, "has_read_scope", lambda u: True)
|
||||
monkeypatch.setattr(plsync, "YouTubeClient", EditedMeanwhile)
|
||||
monkeypatch.setattr(
|
||||
plsync, "_replace_items_from_youtube", lambda d, p, ids: replaced.append(p.id)
|
||||
)
|
||||
|
||||
assert plsync.sync_user_playlists(db, user) == {"synced": 0}
|
||||
assert replaced == []
|
||||
db.expire_all()
|
||||
assert db.get(Playlist, pl.id).name == "L"
|
||||
@@ -0,0 +1,292 @@
|
||||
"""E28 review round 3 — the YouTube sync paths take the playlist row BEFORE they touch an item, and
|
||||
hold it to the commit that writes them.
|
||||
|
||||
Round 2 made every route edit lock the playlist first. The sync paths were left item-first (a push's
|
||||
and a re-pull's item replace, a delete's cascade), so a push beside a remove could deadlock (a 500),
|
||||
and the scheduled mirror refresh re-ingested inside its lock — its commit let go of the lock before
|
||||
the replace, so a remove made then was undone and the list read as synced. A push also replaced the
|
||||
items from a snapshot read before its YouTube calls, and marked the list clean over an edit made
|
||||
during its YouTube writes. The "other tab" here is a second session that commits while the push is
|
||||
talking to YouTube — when the push holds no lock, as a real one does not.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import delete, event, text
|
||||
|
||||
from app.db import SessionLocal
|
||||
from app.models import Channel, Playlist, PlaylistItem, Video
|
||||
from app.sync import playlists as pl_mod
|
||||
|
||||
ITEMS = PlaylistItem.__table__.name
|
||||
LISTS = Playlist.__table__.name
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def trace(db):
|
||||
"""Every statement and every COMMIT on the test engine, in order."""
|
||||
seen: list[str] = []
|
||||
|
||||
def _stmt(conn, cursor, statement, *_):
|
||||
seen.append(" ".join(statement.split()))
|
||||
|
||||
def _commit(conn):
|
||||
seen.append("COMMIT")
|
||||
|
||||
engine = db.get_bind()
|
||||
event.listen(engine, "before_cursor_execute", _stmt)
|
||||
event.listen(engine, "commit", _commit)
|
||||
yield seen
|
||||
event.remove(engine, "before_cursor_execute", _stmt)
|
||||
event.remove(engine, "commit", _commit)
|
||||
|
||||
|
||||
def _held_to_the_item_write(seen: list[str], verb: str) -> bool:
|
||||
"""The first `<verb> … playlist_items` comes after a `FOR UPDATE` of playlists, with no COMMIT
|
||||
in between (a commit ends the lock)."""
|
||||
locked = False
|
||||
for st in seen:
|
||||
if "FOR UPDATE" in st and re.search(rf"FROM {LISTS}\b", st):
|
||||
locked = True
|
||||
elif st == "COMMIT":
|
||||
locked = False
|
||||
elif st.startswith(verb) and re.search(rf"\b{ITEMS}\b", st):
|
||||
return locked
|
||||
return False
|
||||
|
||||
|
||||
class FakeYT:
|
||||
"""YouTube for the sync paths: one playlist, its items, and ids the Data API will not return."""
|
||||
|
||||
def __init__(self, items, *, unavailable=(), title="Mix", on_get=None, on_add=None):
|
||||
self.items = [(f"it{i}", v) for i, v in enumerate(items)]
|
||||
self.unavailable = set(unavailable)
|
||||
self.title = title
|
||||
self.get_calls = 0
|
||||
self._on_get = on_get
|
||||
self._on_add = on_add
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_):
|
||||
return False
|
||||
|
||||
def iter_my_playlists(self):
|
||||
return [{"id": "PLyt", "title": self.title}]
|
||||
|
||||
def iter_my_playlist_video_ids(self, _pid):
|
||||
return [v for _, v in self.items]
|
||||
|
||||
def iter_playlist_items_with_ids(self, _pid):
|
||||
return [{"item_id": i, "video_id": v} for i, v in self.items]
|
||||
|
||||
def get_playlist_snippet(self, _pid):
|
||||
return {"title": self.title}
|
||||
|
||||
def get_videos(self, ids):
|
||||
self.get_calls += 1
|
||||
if self._on_get:
|
||||
self._on_get()
|
||||
return [
|
||||
{"id": v, "snippet": {"channelId": "UCtest", "publishedAt": "2020-01-01T00:00:00Z"}}
|
||||
for v in ids
|
||||
if v not in self.unavailable
|
||||
]
|
||||
|
||||
def create_playlist(self, title, *_, **__):
|
||||
return "PLyt"
|
||||
|
||||
def update_playlist_title(self, _pid, title):
|
||||
self.title = title
|
||||
|
||||
def add_playlist_item(self, _pid, vid, position=None):
|
||||
if self._on_add:
|
||||
self._on_add()
|
||||
iid = f"it{len(self.items) + 100}"
|
||||
self.items.append((iid, vid))
|
||||
return iid
|
||||
|
||||
def delete_playlist_item(self, item_id):
|
||||
self.items = [r for r in self.items if r[0] != item_id]
|
||||
|
||||
def move_playlist_item(self, item_id, _pid, _vid, position):
|
||||
row = next(r for r in self.items if r[0] == item_id)
|
||||
self.items.remove(row)
|
||||
self.items.insert(position, row)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_yt(monkeypatch):
|
||||
def install(fake):
|
||||
monkeypatch.setattr(pl_mod, "YouTubeClient", lambda *_: fake)
|
||||
return fake
|
||||
|
||||
return install
|
||||
|
||||
|
||||
def _seed(db, ids):
|
||||
db.add(Channel(id="UCtest", title="Test"))
|
||||
now = datetime.now(timezone.utc)
|
||||
for v in ids:
|
||||
db.add(Video(id=v, channel_id="UCtest", published_at=now))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _playlist(db, user, local, *, source="local", baseline=None):
|
||||
pl = Playlist(
|
||||
user_id=user.id,
|
||||
name="Mix",
|
||||
source=source,
|
||||
yt_playlist_id="PLyt",
|
||||
synced_video_ids=None if baseline is None else pl_mod._dump_ids(baseline),
|
||||
)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for pos, v in enumerate(local):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=v, position=pos))
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
def _other_tab(pl_id, *, add=None, remove=None, rename=None, drop=False):
|
||||
"""A second session's edit, committed at once — what another tab's request does. It runs on
|
||||
the push's own thread: a lock the push wrongly held here would be waited on forever, so the
|
||||
wait is bounded and a regression fails instead of hanging the gate."""
|
||||
with SessionLocal() as s:
|
||||
s.execute(text("SET LOCAL lock_timeout = '2s'"))
|
||||
if drop:
|
||||
s.execute(delete(Playlist).where(Playlist.id == pl_id))
|
||||
if rename:
|
||||
s.get(Playlist, pl_id).name = rename
|
||||
if add:
|
||||
pos = len(s.query(PlaylistItem).filter_by(playlist_id=pl_id).all())
|
||||
s.add(PlaylistItem(playlist_id=pl_id, video_id=add, position=pos))
|
||||
if remove:
|
||||
s.query(PlaylistItem).filter_by(playlist_id=pl_id, video_id=remove).delete()
|
||||
s.commit()
|
||||
|
||||
|
||||
def test_the_mirror_refresh_holds_its_lock_through_a_list_with_a_private_video(
|
||||
db, make_user, fake_yt, trace
|
||||
):
|
||||
user = make_user()
|
||||
_seed(db, ["a"])
|
||||
pl = _playlist(db, user, ["a"], source="youtube")
|
||||
fake = fake_yt(FakeYT(["a", "ghost"], unavailable={"ghost"}))
|
||||
trace.clear()
|
||||
pl_mod.pull_user_playlists(db, user)
|
||||
assert _held_to_the_item_write(trace, "DELETE"), trace
|
||||
# the private video is asked for once, not again under the lock
|
||||
assert fake.get_calls == 1
|
||||
assert pl_mod._local_video_ids(db, pl) == ["a"]
|
||||
|
||||
|
||||
def test_the_mirror_drop_locks_the_list_before_its_items(db, make_user, fake_yt, trace):
|
||||
user = make_user()
|
||||
_seed(db, ["a"])
|
||||
pl = _playlist(db, user, ["a"], source="youtube")
|
||||
pl.yt_playlist_id = "PLgone"
|
||||
db.commit()
|
||||
fake_yt(FakeYT([]))
|
||||
trace.clear()
|
||||
pl_mod.pull_user_playlists(db, user)
|
||||
assert _held_to_the_item_write(trace, "DELETE"), trace
|
||||
|
||||
|
||||
def test_a_repull_ingests_before_its_lock_and_holds_it_to_the_replace(
|
||||
db, make_user, fake_yt, trace
|
||||
):
|
||||
user = make_user()
|
||||
_seed(db, ["a"])
|
||||
pl = _playlist(db, user, ["a"])
|
||||
fake = fake_yt(FakeYT(["b", "a", "ghost"], unavailable={"ghost"}, title="Renamed"))
|
||||
trace.clear()
|
||||
pl_mod.repull_playlist(db, user, pl)
|
||||
assert _held_to_the_item_write(trace, "DELETE"), trace
|
||||
assert fake.get_calls == 1
|
||||
db.refresh(pl)
|
||||
assert (pl.name, pl.dirty) == ("Renamed", False)
|
||||
assert pl_mod._local_video_ids(db, pl) == ["b", "a"]
|
||||
|
||||
|
||||
def test_a_push_locks_the_list_before_it_replaces_the_items(db, make_user, fake_yt, trace):
|
||||
user = make_user()
|
||||
_seed(db, ["a", "b"])
|
||||
pl = _playlist(db, user, ["a", "b"], baseline=["a"])
|
||||
fake_yt(FakeYT(["a", "y"]))
|
||||
trace.clear()
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
assert _held_to_the_item_write(trace, "DELETE"), trace
|
||||
|
||||
|
||||
def test_an_add_made_while_a_push_reads_youtube_survives_it(db, make_user, fake_yt):
|
||||
user = make_user()
|
||||
_seed(db, ["a", "x"])
|
||||
pl = _playlist(db, user, ["a"], baseline=["a"])
|
||||
# 'y' is new on YouTube, so the push ingests it — and the other tab adds 'x' meanwhile.
|
||||
fake = fake_yt(FakeYT(["a", "y"], on_get=lambda: _other_tab(pl.id, add="x")))
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
local = pl_mod._local_video_ids(db, pl)
|
||||
assert set(local) == {"a", "x", "y"}
|
||||
assert "x" in [v for _, v in fake.items] # and it reached YouTube as a local add
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linked", [True, False], ids=["linked", "first-push"])
|
||||
def test_a_rename_made_during_a_push_s_youtube_writes_stays_dirty(db, make_user, fake_yt, linked):
|
||||
"""The closing baseline is YouTube's title, not the one the lock re-reads: over the new name it
|
||||
matched the rename, and the list read synced with YouTube still on the old title."""
|
||||
user = make_user()
|
||||
_seed(db, ["a", "b"])
|
||||
pl = _playlist(db, user, ["a", "b"], baseline=["a"] if linked else None)
|
||||
if not linked:
|
||||
pl.yt_playlist_id = None
|
||||
db.commit()
|
||||
fake = fake_yt(
|
||||
FakeYT(["a"] if linked else [], on_add=lambda: _other_tab(pl.id, rename="Mix 2"))
|
||||
)
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
db.refresh(pl)
|
||||
assert (pl.name, fake.title) == ("Mix 2", "Mix")
|
||||
assert pl.dirty is True # the rename has not reached YouTube; the list must say so
|
||||
|
||||
|
||||
def test_a_push_stops_at_a_list_deleted_while_it_read_youtube(db, make_user, fake_yt):
|
||||
"""A delete landing mid-push left a plan over no items: every item "removed here", so the push
|
||||
emptied the YouTube playlist the delete meant to keep."""
|
||||
user = make_user()
|
||||
_seed(db, ["a", "b"])
|
||||
pl = _playlist(db, user, ["a", "b"])
|
||||
# a video YouTube would not return last time, retried now — the read the delete lands in
|
||||
pl.synced_video_ids = json.dumps({"ids": ["a", "b"], "unavailable": ["ghost"]})
|
||||
db.commit()
|
||||
fake = fake_yt(
|
||||
FakeYT(
|
||||
["a", "b", "ghost"], unavailable={"ghost"}, on_get=lambda: _other_tab(pl.id, drop=True)
|
||||
)
|
||||
)
|
||||
raised = None
|
||||
try:
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
except Exception as exc: # noqa: BLE001 — the regression raises too, AFTER its YouTube writes
|
||||
raised = exc
|
||||
assert [v for _, v in fake.items] == ["a", "b", "ghost"] # YouTube untouched
|
||||
assert isinstance(raised, pl_mod.PlaylistGone) # which the route answers with a 404
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linked", [True, False], ids=["linked", "first-push"])
|
||||
def test_an_edit_made_during_a_push_s_youtube_writes_stays_dirty(db, make_user, fake_yt, linked):
|
||||
user = make_user()
|
||||
_seed(db, ["a", "b"])
|
||||
pl = _playlist(db, user, ["a", "b"], baseline=["a"] if linked else None)
|
||||
if not linked:
|
||||
pl.yt_playlist_id = None
|
||||
db.commit()
|
||||
fake_yt(FakeYT(["a"] if linked else [], on_add=lambda: _other_tab(pl.id, remove="a")))
|
||||
pl_mod.merge_playlist(db, user, pl)
|
||||
db.refresh(pl)
|
||||
assert pl_mod._local_video_ids(db, pl) == ["b"]
|
||||
assert pl.dirty is True # the removal has not reached YouTube; the list must say so
|
||||
@@ -11,21 +11,21 @@ from app.plex.cards import movie_card, show_card
|
||||
|
||||
def test_movie_card_emits_art_when_art_key_present():
|
||||
it = PlexItem(rating_key="m1", kind="movie", title="M", year=2020, art_key="/library/art/1")
|
||||
card = movie_card(it, None)
|
||||
card = movie_card(it, None, lang="en")
|
||||
assert card["thumb"] == "/api/plex/image/m1"
|
||||
assert card["art"] == "/api/plex/image/m1?variant=art"
|
||||
|
||||
|
||||
def test_movie_card_art_is_none_without_art_key():
|
||||
it = PlexItem(rating_key="m2", kind="movie", title="M2")
|
||||
assert movie_card(it, None)["art"] is None
|
||||
assert movie_card(it, None, lang="en")["art"] is None
|
||||
|
||||
|
||||
def test_show_card_emits_art_when_art_key_present():
|
||||
sh = PlexShow(rating_key="s1", title="S", art_key="/library/art/2")
|
||||
assert show_card(sh)["art"] == "/api/plex/image/s1?variant=art"
|
||||
assert show_card(sh, lang="en")["art"] == "/api/plex/image/s1?variant=art"
|
||||
|
||||
|
||||
def test_show_card_art_is_none_without_art_key():
|
||||
sh = PlexShow(rating_key="s2", title="S2")
|
||||
assert show_card(sh)["art"] is None
|
||||
assert show_card(sh, lang="en")["art"] is None
|
||||
|
||||
@@ -352,3 +352,52 @@ def test_show_detail_lazy_fills_when_unenriched(db, monkeypatch, override_dep):
|
||||
assert show.imdb_id == "tt0068646" # persisted for next time
|
||||
assert show.people_enriched_at is not None
|
||||
assert [c["name"] for c in show.full_cast] == ["Marlon Brando", "Al Pacino"]
|
||||
|
||||
|
||||
def test_item_detail_names_tracks_from_their_language_not_plexs_title(
|
||||
db, monkeypatch, override_dep
|
||||
):
|
||||
"""E30: Plex titles a track in the Plex SERVER's language ("English (AC3 5.1)", "magyar Forced",
|
||||
"中文 (EAC3 Stereo)"), so a Hungarian reader got a mix. The route sends what the client names it
|
||||
from instead — the language tag, the forced / SDH flags, the codec tail — keeping Plex's title
|
||||
only as the fallback, and no invented English "Audio 2" for a track with neither."""
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
db.add(PlexItem(rating_key="m1", library_id=lib.id, kind="movie", title="T"))
|
||||
db.commit()
|
||||
user = _user(db, "tracks@example.com")
|
||||
db.commit()
|
||||
streams = [
|
||||
{"streamType": 2, "displayTitle": "English (AC3 5.1)", "languageTag": "en-US"},
|
||||
{"streamType": 2, "displayTitle": "中文 (EAC3 Stereo)", "languageTag": "zh"},
|
||||
{"streamType": 2},
|
||||
{
|
||||
"streamType": 3,
|
||||
"displayTitle": "magyar Forced",
|
||||
"languageTag": "hu",
|
||||
"forced": True,
|
||||
"codec": "srt",
|
||||
},
|
||||
{
|
||||
"streamType": 3,
|
||||
"displayTitle": "English SDH",
|
||||
"languageTag": "en",
|
||||
"hearingImpaired": True,
|
||||
"codec": "srt",
|
||||
},
|
||||
]
|
||||
meta = {**RICH_META, "Media": [{"Part": [{"Stream": streams}]}]}
|
||||
monkeypatch.setattr(detail_routes, "PlexClient", lambda db: _StubClient(meta))
|
||||
body = _get(db, user, monkeypatch, override_dep, "/api/plex/item/m1").json()
|
||||
|
||||
naming = ("label", "lang_tag", "forced", "hearing_impaired", "detail")
|
||||
assert [tuple(a[k] for k in naming) for a in body["audio_streams"]] == [
|
||||
("English (AC3 5.1)", "en-US", False, False, "AC3 5.1"),
|
||||
("中文 (EAC3 Stereo)", "zh", False, False, "EAC3 Stereo"),
|
||||
(None, None, False, False, None),
|
||||
]
|
||||
assert [tuple(s[k] for k in naming) for s in body["subtitle_streams"]] == [
|
||||
("magyar Forced", "hu", True, False, None),
|
||||
("English SDH", "en", False, True, None),
|
||||
]
|
||||
|
||||
@@ -12,7 +12,19 @@ import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import PlexItem, PlexLibrary, User
|
||||
from app import sysconfig
|
||||
from app.plex.cards import display_summary, display_title
|
||||
from app.models import (
|
||||
PlexCollection,
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
PlexPlaylist,
|
||||
PlexPlaylistItem,
|
||||
PlexSeason,
|
||||
PlexShow,
|
||||
PlexState,
|
||||
User,
|
||||
)
|
||||
|
||||
|
||||
def _seed(db):
|
||||
@@ -30,7 +42,9 @@ def _seed(db):
|
||||
summary="English summary.",
|
||||
summary_hu="Magyar összefoglaló.",
|
||||
genres=["Drama"],
|
||||
genres_hu=["Dráma"],
|
||||
genres_hu=["Dráma", "Akció"],
|
||||
tagline="English tagline.",
|
||||
tagline_hu="Magyar mottó.",
|
||||
rating=6.5,
|
||||
imdb_rating=None,
|
||||
year=2001,
|
||||
@@ -44,7 +58,9 @@ def _seed(db):
|
||||
library_id=lib.id,
|
||||
kind="movie",
|
||||
title="EN Only",
|
||||
genres=["Comedy"], # no HU twin
|
||||
summary="English only.",
|
||||
tagline="English only tagline.",
|
||||
genres=["Comedy"], # no HU twin, never reached by a sync since: no genres_hu
|
||||
year=2002,
|
||||
added_at=datetime(2024, 1, 2, tzinfo=timezone.utc),
|
||||
people_enriched_at=datetime(2024, 1, 2, tzinfo=timezone.utc),
|
||||
@@ -102,12 +118,52 @@ def test_genre_filter_matches_active_language_vocabulary(db, monkeypatch, overri
|
||||
assert miss.json()["items"] == []
|
||||
|
||||
|
||||
def test_a_genre_filter_saved_in_english_still_matches_in_hungarian(db, monkeypatch, override_dep):
|
||||
"""E30: the HU column holds one Hungarian word per genre. A filter saved while the HU list still
|
||||
offered "Action" is read through the same map, so it finds the "Akció" title instead of nothing."""
|
||||
user = _seed(db)
|
||||
resp = _get(
|
||||
db, user, monkeypatch, override_dep, "/api/plex/library?scope=movie&lang=hu&genres=Action"
|
||||
)
|
||||
assert _titles(resp) == {"Magyar Cím"}
|
||||
|
||||
|
||||
def test_a_genre_the_hungarian_list_offers_in_english_still_matches(db, monkeypatch, override_dep):
|
||||
"""E30 review: a row the sync has not given Hungarian genres yet is listed under its English word
|
||||
(the facets read coalesce(genres_hu, genres)); filtering by that word must find it."""
|
||||
user = _seed(db)
|
||||
facets = _get(db, user, monkeypatch, override_dep, "/api/plex/facets?scope=movie&lang=hu")
|
||||
assert "Comedy" in {g["value"] for g in facets.json()["genres"]}
|
||||
resp = _get(
|
||||
db, user, monkeypatch, override_dep, "/api/plex/library?scope=movie&lang=hu&genres=Comedy"
|
||||
)
|
||||
assert _titles(resp) == {"EN Only"}
|
||||
|
||||
|
||||
def test_a_title_without_a_hungarian_twin_reads_in_english(db, monkeypatch, override_dep):
|
||||
user = _seed(db)
|
||||
body = _get(db, user, monkeypatch, override_dep, "/api/plex/item/m2?lang=hu").json()
|
||||
assert (body["title"], body["summary"], body["tagline"]) == (
|
||||
"EN Only",
|
||||
"English only.",
|
||||
"English only tagline.",
|
||||
)
|
||||
row = PlexItem(title="T", summary="S")
|
||||
assert (display_title(row, "hu"), display_summary(row, "hu")) == ("T", "S")
|
||||
row.title_hu, row.summary_hu = "HT", "HS"
|
||||
assert (display_title(row, "hu"), display_summary(row, "hu")) == ("HT", "HS")
|
||||
assert (display_title(row, "en"), display_summary(row, "en")) == ("T", "S")
|
||||
|
||||
|
||||
def test_item_detail_language_and_rating_split(db, monkeypatch, override_dep):
|
||||
user = _seed(db)
|
||||
resp = _get(db, user, monkeypatch, override_dep, "/api/plex/item/m1?lang=hu")
|
||||
body = resp.json()
|
||||
assert body["title"] == "Magyar Cím"
|
||||
assert body["summary"] == "Magyar összefoglaló."
|
||||
assert body["tagline"] == "Magyar mottó." # E30: the HU overlay's tagline
|
||||
en = _get(db, user, monkeypatch, override_dep, "/api/plex/item/m1").json()
|
||||
assert en["tagline"] == "English tagline."
|
||||
# imdb_rating is IMDb-only (None here); the audience score is served separately as `rating`.
|
||||
assert body["imdb_rating"] is None
|
||||
assert body["rating"] == 6.5
|
||||
@@ -147,3 +203,126 @@ def test_rating_filter_prefers_imdb_over_audience(db, monkeypatch, override_dep)
|
||||
assert "High IMDb" in _titles(hit)
|
||||
miss = _get(db, user, monkeypatch, override_dep, "/api/plex/library?scope=movie&rating_min=9")
|
||||
assert "High IMDb" not in _titles(miss)
|
||||
|
||||
|
||||
def _seed_rails(db):
|
||||
"""`_seed` plus what every card rail reads: a HU-overlaid show with a HU-overlaid episode in
|
||||
progress, a cast name, a collection, and a playlist holding the movie and the episode."""
|
||||
user = _seed(db)
|
||||
m1 = db.query(PlexItem).filter_by(rating_key="m1").one()
|
||||
m1.cast_names = ["Anna"]
|
||||
m1.collection_keys = ["c1"]
|
||||
db.add(PlexCollection(rating_key="c1", library_id=m1.library_id, title="Coll"))
|
||||
tv = PlexLibrary(plex_key="2", title="TV Shows", kind="show", enabled=True)
|
||||
db.add(tv)
|
||||
db.flush()
|
||||
show = PlexShow(
|
||||
rating_key="s1",
|
||||
library_id=tv.id,
|
||||
title="English Show",
|
||||
title_hu="Magyar Sorozat",
|
||||
added_at=datetime(2024, 1, 3, tzinfo=timezone.utc),
|
||||
)
|
||||
db.add(show)
|
||||
db.flush()
|
||||
season = PlexSeason(rating_key="s1s1", show_id=show.id, title="Season 1", season_number=1)
|
||||
db.add(season)
|
||||
db.flush()
|
||||
ep = PlexItem(
|
||||
rating_key="e1",
|
||||
library_id=tv.id,
|
||||
kind="episode",
|
||||
show_id=show.id,
|
||||
season_id=season.id,
|
||||
title="Pilot",
|
||||
title_hu="Kezdet",
|
||||
season_number=1,
|
||||
episode_number=1,
|
||||
duration_s=3000,
|
||||
)
|
||||
db.add(ep)
|
||||
db.flush()
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=ep.id,
|
||||
status="in_progress",
|
||||
position_seconds=600,
|
||||
progress_updated_at=datetime(2024, 2, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
)
|
||||
pl = PlexPlaylist(user_id=user.id, title="Mix")
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
db.add_all(
|
||||
[
|
||||
PlexPlaylistItem(playlist_id=pl.id, item_id=m1.id, position=0),
|
||||
PlexPlaylistItem(playlist_id=pl.id, item_id=ep.id, position=1),
|
||||
]
|
||||
)
|
||||
# TMDB extras: m2 recommends m1 (held) and a title we lack; m1, watched, anchors the recs rail.
|
||||
m1.guid = "tmdb:1"
|
||||
m1.tmdb_enriched_at = datetime(2024, 1, 1, tzinfo=timezone.utc)
|
||||
m1.recommendations = [{"tmdb_id": 2, "title": "EN Only", "media_type": "movie"}]
|
||||
m2 = db.query(PlexItem).filter_by(rating_key="m2").one()
|
||||
m2.guid = "tmdb:2"
|
||||
m2.tmdb_enriched_at = datetime(2024, 1, 1, tzinfo=timezone.utc)
|
||||
m2.recommendations = [
|
||||
{"tmdb_id": 1, "title": "English Title", "media_type": "movie"},
|
||||
{"tmdb_id": 77, "title": "Elsewhere", "media_type": "movie"},
|
||||
]
|
||||
db.add(
|
||||
PlexState(
|
||||
user_id=user.id,
|
||||
item_id=m1.id,
|
||||
status="watched",
|
||||
watched_at=datetime(2024, 3, 1, tzinfo=timezone.utc),
|
||||
)
|
||||
)
|
||||
sysconfig.set_value(db, "plex_enabled", True)
|
||||
db.commit()
|
||||
return user, pl.id
|
||||
|
||||
|
||||
def test_every_card_rail_serves_the_readers_language(db, monkeypatch, override_dep):
|
||||
"""E30: the grid spoke Hungarian, but every other card surface (Discover rails, show page,
|
||||
search, collection, person page, playlist, Home) served the English title to a HU reader."""
|
||||
user, pid = _seed_rails(db)
|
||||
|
||||
def get(path):
|
||||
return _get(db, user, monkeypatch, override_dep, path).json()
|
||||
|
||||
def titles(cards):
|
||||
return {c["title"] for c in cards}
|
||||
|
||||
rails = get("/api/plex/discover?scope=both&lang=hu")
|
||||
assert {"Magyar Cím", "Magyar Sorozat"} <= titles(rails["recently_added"])
|
||||
assert "Magyar Sorozat" in titles(rails["continue_watching"])
|
||||
assert titles(get("/api/plex/show/s1?lang=hu")["seasons"][0]["episodes"]) == {"Kezdet"}
|
||||
found = get("/api/plex/library?scope=both&lang=hu&q=Kezdet")["episodes"]
|
||||
assert [(e["title"], e["show_title"]) for e in found] == [("Kezdet", "Magyar Sorozat")]
|
||||
assert titles(get("/api/plex/collections/c1?lang=hu")["items"]) == {"Magyar Cím"}
|
||||
assert titles(get("/api/plex/person/Anna?lang=hu")["movies"]) == {"Magyar Cím"}
|
||||
items = get(f"/api/plex/playlists/{pid}?lang=hu")["items"]
|
||||
assert [(c["title"], c.get("show_title")) for c in items] == [
|
||||
("Magyar Cím", None),
|
||||
("Kezdet", "Magyar Sorozat"),
|
||||
]
|
||||
# A recommendation we hold is named as the library names it; the rail's anchor too.
|
||||
recs = get("/api/plex/item/m2?lang=hu")["tmdb"]["recommendations"]
|
||||
assert [(r["title"], r["in_library"]) for r in recs] == [
|
||||
("Magyar Cím", True),
|
||||
("Elsewhere", False),
|
||||
]
|
||||
byw = get("/api/plex/discover?scope=both&lang=hu")["because_you_watched"]
|
||||
assert byw["anchor"]["title"] == "Magyar Cím"
|
||||
home = get("/api/discover?lang=hu")["continue_watching"]
|
||||
plex = [c for c in home if c["source"] == "plex"]
|
||||
assert [c["title"] for c in plex] == ["Magyar Sorozat"]
|
||||
assert plex[0]["next"]["title"] == "Kezdet"
|
||||
# EN (the default) still serves English on the same rails.
|
||||
items = get(f"/api/plex/playlists/{pid}")["items"]
|
||||
assert [(c["title"], c.get("show_title")) for c in items] == [
|
||||
("English Title", None),
|
||||
("Pilot", "English Show"),
|
||||
]
|
||||
|
||||
@@ -6,11 +6,16 @@ Uses the DB lane (`db` fixture) — skipped when no Postgres is reachable. Runs
|
||||
(migrations 0063/0064) so the JSONB + STORED generated columns are exercised.
|
||||
"""
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
import app.plex.sync as syncmod
|
||||
from app import sysconfig
|
||||
from app.models import PlexItem, PlexLibrary
|
||||
from app.db import engine
|
||||
from app.models import PlexItem, PlexLibrary, PlexShow
|
||||
from app.plex.genres import hu_genre, hu_genres
|
||||
from app.plex.sync import _apply_item, _canonical_guid, _sync_overlay
|
||||
from app.plex.query import content_libs
|
||||
|
||||
@@ -244,3 +249,171 @@ def test_content_libs_excludes_overlay_libraries(db):
|
||||
assert en_lib.id in movie_ids
|
||||
assert overlay.id not in movie_ids # the HU overlay is never browsable
|
||||
assert show_ids == [en_shows.id]
|
||||
|
||||
|
||||
class _KindStub:
|
||||
"""An overlay section listing per Plex type (2 = show, 4 = episode), like a HU TV section."""
|
||||
|
||||
def __init__(self, by_type: dict[int, list[dict]]):
|
||||
self._by = by_type
|
||||
|
||||
def section_items(self, key, item_type, start, size):
|
||||
items = self._by.get(item_type, [])
|
||||
return items[start : start + size], len(items)
|
||||
|
||||
|
||||
def test_a_show_overlay_writes_hungarian_episode_titles_and_summaries(db):
|
||||
"""E30: the HU TV section lists every episode in Hungarian; they were never read, so a Hungarian
|
||||
reader saw every episode title and summary in English."""
|
||||
en_lib = PlexLibrary(plex_key="1", title="TV Shows", kind="show")
|
||||
hu_lib = PlexLibrary(plex_key="9", title="Sorozatok", kind="show", metadata_lang="hu")
|
||||
db.add_all([en_lib, hu_lib])
|
||||
db.flush()
|
||||
show = PlexShow(rating_key="s1", library_id=en_lib.id, title="3 Body Problem", guid="tmdb:10")
|
||||
db.add(show)
|
||||
db.flush()
|
||||
ep = PlexItem(
|
||||
rating_key="e1",
|
||||
library_id=en_lib.id,
|
||||
kind="episode",
|
||||
show_id=show.id,
|
||||
title="Countdown",
|
||||
summary="EN.",
|
||||
guid="tmdb:501",
|
||||
)
|
||||
db.add(ep)
|
||||
db.commit()
|
||||
stub = _KindStub(
|
||||
{
|
||||
syncmod._TYPE_SHOW: [_hu_movie("hs", "A 3-test-probléma", "HU.", [], "tmdb://10")],
|
||||
syncmod._TYPE_EPISODE: [
|
||||
_hu_movie("he", "Visszaszámlálás", "Magyar epizód.", [], "tmdb://501")
|
||||
],
|
||||
}
|
||||
)
|
||||
_sync_overlay(db, stub, hu_lib, en_lib.id, {})
|
||||
db.commit()
|
||||
db.refresh(ep)
|
||||
db.refresh(show)
|
||||
assert (ep.title_hu, ep.summary_hu) == ("Visszaszámlálás", "Magyar epizód.")
|
||||
assert show.title_hu == "A 3-test-probléma"
|
||||
|
||||
|
||||
def test_the_overlay_gives_every_row_one_hungarian_word_per_genre(db):
|
||||
"""E30: Plex's HU agent mixes "Action" with "Dráma", and a title with no HU twin kept only its
|
||||
English list — so the HU genre list offered "Drama" beside "Dráma"."""
|
||||
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||
hu_lib = PlexLibrary(plex_key="9", title="Filmek", kind="movie", metadata_lang="hu")
|
||||
db.add_all([en_lib, hu_lib])
|
||||
db.flush()
|
||||
twin = PlexItem(
|
||||
rating_key="m1",
|
||||
library_id=en_lib.id,
|
||||
kind="movie",
|
||||
title="Heat",
|
||||
guid="tmdb:1",
|
||||
genres=["Action", "Crime"],
|
||||
)
|
||||
lone = PlexItem(
|
||||
rating_key="m2",
|
||||
library_id=en_lib.id,
|
||||
kind="movie",
|
||||
title="Solo",
|
||||
guid="tmdb:2",
|
||||
genres=["Drama", "Science Fiction"],
|
||||
)
|
||||
db.add_all([twin, lone])
|
||||
db.commit()
|
||||
meta = _hu_movie(
|
||||
"h1", "Szemtől szemben", "HU.", ["Action", "Bűnügyi", "Drama", "Dráma"], "tmdb://1"
|
||||
)
|
||||
meta["tagline"] = "Magyar mottó."
|
||||
_sync_overlay(db, _StubPlex([meta]), hu_lib, en_lib.id, {})
|
||||
db.commit()
|
||||
db.refresh(twin)
|
||||
db.refresh(lone)
|
||||
assert twin.genres_hu == ["Akció", "Bűnügyi", "Dráma"]
|
||||
assert twin.tagline_hu == "Magyar mottó."
|
||||
assert lone.genres_hu == ["Dráma", "Sci-Fi"] # no twin: its own English list, in Hungarian
|
||||
assert lone.tagline_hu is None
|
||||
|
||||
|
||||
def test_a_twin_without_genres_keeps_the_rows_own_hungarian_list(db):
|
||||
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||
hu_lib = PlexLibrary(plex_key="9", title="Filmek", kind="movie", metadata_lang="hu")
|
||||
db.add_all([en_lib, hu_lib])
|
||||
db.flush()
|
||||
row = PlexItem(
|
||||
rating_key="m1",
|
||||
library_id=en_lib.id,
|
||||
kind="movie",
|
||||
title="Heat",
|
||||
guid="tmdb:1",
|
||||
genres=["Crime"],
|
||||
)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
_sync_overlay(
|
||||
db,
|
||||
_StubPlex([_hu_movie("h1", "Szemtől szemben", "HU.", [], "tmdb://1")]),
|
||||
hu_lib,
|
||||
en_lib.id,
|
||||
{},
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
assert (row.title_hu, row.genres_hu) == ("Szemtől szemben", ["Bűnügyi"])
|
||||
|
||||
|
||||
def test_every_synced_row_gets_hungarian_genres():
|
||||
"""E30 review: not only an overlaid row — the HU facets and filter read genres_hu for all."""
|
||||
from app.plex.sync import _apply_facet_fields
|
||||
|
||||
row = PlexItem(title="Solo")
|
||||
_apply_facet_fields(row, {"Genre": [{"tag": "Drama"}, {"tag": "Action"}]})
|
||||
assert (row.genres, row.genres_hu) == (["Drama", "Action"], ["Dráma", "Akció"])
|
||||
|
||||
|
||||
def test_the_genre_map_is_idempotent_and_keeps_an_unknown_word():
|
||||
assert hu_genre("science fiction") == "Sci-Fi" # case-insensitive
|
||||
assert hu_genre("Dráma") == "Dráma"
|
||||
assert hu_genre("Space Opera") == "Space Opera" # unknown: shown as Plex gave it, never dropped
|
||||
once = hu_genres(["Drama", "Action", "Dráma"])
|
||||
assert once == ["Dráma", "Akció"]
|
||||
assert hu_genres(once) == once
|
||||
assert hu_genres(None) is None and hu_genres([]) is None
|
||||
|
||||
|
||||
def test_0094_gives_every_row_without_a_hungarian_list_one(db):
|
||||
cfg = Config("alembic.ini")
|
||||
command.downgrade(cfg, "0093_plex_person_known_for_votes")
|
||||
upgraded = False
|
||||
try:
|
||||
with Session(engine) as s:
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
s.add(lib)
|
||||
s.flush()
|
||||
rows = {
|
||||
"bare": PlexItem(genres=["Action", "Biography"]),
|
||||
"overlaid": PlexItem(
|
||||
genres=["Action"], genres_hu=["Kaland"]
|
||||
), # its twin's own list
|
||||
"genreless": PlexItem(genres=[]),
|
||||
}
|
||||
for key, row in rows.items():
|
||||
row.rating_key, row.library_id, row.kind, row.title = key, lib.id, "movie", key
|
||||
show = PlexShow(rating_key="s", library_id=lib.id, title="s", genres=["Comedy"])
|
||||
s.add_all([*rows.values(), show])
|
||||
s.commit()
|
||||
ids = {k: r.id for k, r in rows.items()}
|
||||
show_id = show.id
|
||||
command.upgrade(cfg, "head")
|
||||
upgraded = True
|
||||
with Session(engine) as s:
|
||||
got = {k: s.get(PlexItem, i).genres_hu for k, i in ids.items()} # type: ignore[union-attr]
|
||||
show_hu = s.get(PlexShow, show_id).genres_hu # type: ignore[union-attr]
|
||||
assert got == {"bare": ["Akció", "Életrajzi"], "overlaid": ["Kaland"], "genreless": None}
|
||||
assert show_hu == ["Vígjáték"]
|
||||
finally:
|
||||
if not upgraded:
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
@@ -0,0 +1,463 @@
|
||||
"""E30 S6: the people are made local ahead of time. The people_enrich job fetches linked people's TMDB
|
||||
bios (the most-credited first) and caches their portraits resized through Plex; the person page says
|
||||
when a bio is still on its way (`enriching`), so it can ask again instead of staying empty.
|
||||
DB lane (Postgres); the portrait cache runs in a temp directory.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app import scheduler
|
||||
from app.models import PlexItem, PlexLibrary, PlexPerson, User
|
||||
from app.plex import imgcache
|
||||
from app.plex import people as plex_people
|
||||
from app.plex import tmdb as plex_tmdb
|
||||
|
||||
CDN = plex_people.PERSON_IMG_HOST + "people/"
|
||||
DONE = datetime(2024, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _movie(db, lib, key, cast):
|
||||
db.add(PlexItem(rating_key=key, library_id=lib.id, kind="movie", title=key, cast_names=cast))
|
||||
|
||||
|
||||
def _library(db):
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
return lib
|
||||
|
||||
|
||||
class _Tmdb:
|
||||
"""A TmdbClient stand-in: records the ids asked for, fails on `fail_on`."""
|
||||
|
||||
def __init__(self, fail_on=()):
|
||||
self.asked: list[int] = []
|
||||
self.fail_on = set(fail_on)
|
||||
|
||||
def fetch_person(self, pid):
|
||||
self.asked.append(pid)
|
||||
if pid in self.fail_on:
|
||||
raise plex_tmdb.TmdbError("429")
|
||||
return {"biography": f"Bio {pid}."}
|
||||
|
||||
|
||||
def test_the_batch_takes_the_most_credited_linked_people_first(db):
|
||||
lib = _library(db)
|
||||
_movie(db, lib, "a", ["Often", "Sometimes"])
|
||||
_movie(db, lib, "b", ["Often", "Sometimes"])
|
||||
_movie(db, lib, "c", ["Often", "Unlinked"])
|
||||
db.add_all(
|
||||
[
|
||||
PlexPerson(name="Sometimes", tmdb_person_id=2),
|
||||
PlexPerson(name="Often", tmdb_person_id=1),
|
||||
PlexPerson(name="Unlinked"), # no TMDB id: nothing to fetch
|
||||
PlexPerson(name="Done", tmdb_person_id=3, tmdb_enriched_at=DONE),
|
||||
PlexPerson(name="Rare", tmdb_person_id=4), # credited nowhere: last
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
tmdb = _Tmdb()
|
||||
assert plex_tmdb.enrich_people_batch(db, tmdb, 2) == {"enriched": 2} # type: ignore[arg-type]
|
||||
assert tmdb.asked == [1, 2]
|
||||
db.expire_all()
|
||||
often = db.query(PlexPerson).filter_by(name="Often").one()
|
||||
assert often.biography == "Bio 1." and often.tmdb_enriched_at is not None
|
||||
assert db.query(PlexPerson).filter_by(name="Rare").one().tmdb_enriched_at is None # next run
|
||||
|
||||
|
||||
def test_a_failed_lookup_is_skipped_and_the_rest_still_land(db):
|
||||
db.add_all([PlexPerson(name="A", tmdb_person_id=1), PlexPerson(name="B", tmdb_person_id=2)])
|
||||
db.add(PlexPerson(name="C", tmdb_person_id=3))
|
||||
db.commit()
|
||||
tmdb = _Tmdb(fail_on={2})
|
||||
assert plex_tmdb.enrich_people_batch(db, tmdb, 10) == {"enriched": 2, "failed": 1} # type: ignore[arg-type]
|
||||
assert tmdb.asked == [1, 2, 3] # B refused: skipped, not the end of the batch
|
||||
db.expire_all()
|
||||
stamped = {p.name: p.tmdb_enriched_at is not None for p in db.query(PlexPerson)}
|
||||
assert stamped == {"A": True, "B": False, "C": True} # B is tried again next run
|
||||
|
||||
|
||||
def test_failures_in_a_row_end_the_batch(db):
|
||||
db.add_all([PlexPerson(name=f"P{i}", tmdb_person_id=i) for i in range(1, 7)])
|
||||
db.commit()
|
||||
tmdb = _Tmdb(fail_on={1, 2, 3, 4, 5, 6}) # TMDB is down
|
||||
assert plex_tmdb.enrich_people_batch(db, tmdb, 10) == {"enriched": 0, "failed": 3} # type: ignore[arg-type]
|
||||
assert tmdb.asked == [1, 2, 3]
|
||||
|
||||
|
||||
def test_failures_between_successes_do_not_end_the_batch(db):
|
||||
db.add_all([PlexPerson(name=f"P{i}", tmdb_person_id=i) for i in range(1, 9)])
|
||||
db.commit()
|
||||
tmdb = _Tmdb(fail_on={1, 3, 5, 7}) # fail, ok, fail, ok, …: never three in a row
|
||||
assert plex_tmdb.enrich_people_batch(db, tmdb, 10) == {"enriched": 4, "failed": 4} # type: ignore[arg-type]
|
||||
assert tmdb.asked == list(range(1, 9))
|
||||
|
||||
|
||||
def test_the_batch_stores_the_hungarian_biography(db):
|
||||
db.add(PlexPerson(name="A", tmdb_person_id=1))
|
||||
db.commit()
|
||||
|
||||
class _WithHu(_Tmdb):
|
||||
def fetch_person(self, pid):
|
||||
return {
|
||||
"biography": "English.",
|
||||
"translations": {
|
||||
"translations": [{"iso_639_1": "hu", "data": {"biography": "Magyar."}}]
|
||||
},
|
||||
}
|
||||
|
||||
assert plex_tmdb.enrich_people_batch(db, _WithHu(), 10) == {"enriched": 1} # type: ignore[arg-type]
|
||||
db.expire_all()
|
||||
person = db.query(PlexPerson).one()
|
||||
assert (person.biography, person.biography_hu) == ("English.", "Magyar.")
|
||||
|
||||
|
||||
class _Plex:
|
||||
"""A PlexClient stand-in. `down`: urls Plex does not answer for (True: none); `dead`: urls Plex
|
||||
answers `gone` for (a dead CDN file: "not found"); `status`: an error status Plex answers every
|
||||
other url with."""
|
||||
|
||||
def __init__(self, down=False, dead=(), status=None, gone=404):
|
||||
self.asked: list[tuple[str, int | None]] = []
|
||||
self.down = down
|
||||
self.dead = set(dead)
|
||||
self.status = status
|
||||
self.gone = gone
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def image_bytes(self, url, width=None, height=None):
|
||||
self.asked.append((url, width))
|
||||
if self.down is True or (self.down and url in self.down):
|
||||
raise plex_people.PlexError("down") from httpx.ConnectError("refused")
|
||||
code = self.gone if url in self.dead else self.status
|
||||
if code:
|
||||
req = httpx.Request("GET", url)
|
||||
status = httpx.HTTPStatusError(
|
||||
str(code), request=req, response=httpx.Response(code, request=req)
|
||||
)
|
||||
raise plex_people.PlexError(str(code)) from status
|
||||
return b"small", "image/jpeg"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cache_dir(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(imgcache, "CACHE_DIR", tmp_path)
|
||||
return tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plex_on(monkeypatch):
|
||||
"""The Plex module switched on (the job's gate), with `plex` as its client."""
|
||||
state = {"enabled": True, "plex": _Plex()}
|
||||
monkeypatch.setattr(plex_people.sysconfig, "get_bool", lambda d, k: state["enabled"])
|
||||
monkeypatch.setattr(plex_people, "PlexClient", lambda d: state["plex"])
|
||||
return state
|
||||
|
||||
|
||||
def _faces(db, n):
|
||||
db.add_all(
|
||||
[
|
||||
PlexPerson(name=f"P{i}", photo_thumb=plex_people.person_thumb_url(CDN + f"{i}.jpg"))
|
||||
for i in range(n)
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def test_portraits_are_cached_small_and_replace_the_full_size_original(db, cache_dir, plex_on):
|
||||
lib = _library(db)
|
||||
_movie(db, lib, "a", ["Often"])
|
||||
_movie(db, lib, "b", ["Often", "Cached"])
|
||||
db.add_all(
|
||||
[
|
||||
PlexPerson(name="Often", photo_thumb=plex_people.person_thumb_url(CDN + "often.jpg")),
|
||||
PlexPerson(name="Cached", photo_thumb=plex_people.person_thumb_url(CDN + "cached.jpg")),
|
||||
PlexPerson(name="Faceless"),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
small, original = plex_people._portrait_keys(CDN + "often.jpg")
|
||||
imgcache.write(original, b"a big original", "image/jpeg") # cached without Plex
|
||||
imgcache.write(plex_people._portrait_keys(CDN + "cached.jpg")[0], b"x", "image/jpeg")
|
||||
plex = plex_on["plex"]
|
||||
|
||||
assert plex_people.warm_portraits(db, 10) == {"cached": 1}
|
||||
assert plex.asked == [
|
||||
(CDN + "often.jpg", plex_people.PORTRAIT_BOX)
|
||||
] # resized, the cached one skipped
|
||||
assert imgcache.read(small) == (b"small", "image/jpeg")
|
||||
assert not imgcache.has(original)
|
||||
|
||||
|
||||
def test_portraits_wait_for_plex(db, cache_dir, plex_on, monkeypatch):
|
||||
_faces(db, 1)
|
||||
|
||||
def not_configured(d):
|
||||
raise plex_people.PlexNotConfigured()
|
||||
|
||||
monkeypatch.setattr(plex_people, "PlexClient", not_configured)
|
||||
assert plex_people.warm_portraits(db, 10) == {"cached": 0, "skipped": "disabled"}
|
||||
assert list(cache_dir.iterdir()) == [] # no full-size originals behind Plex's back
|
||||
|
||||
|
||||
def test_portraits_wait_for_the_plex_module_to_be_on(db, cache_dir, plex_on):
|
||||
_faces(db, 1)
|
||||
plex_on["enabled"] = False # switched off, its server still set up
|
||||
assert plex_people.warm_portraits(db, 10) == {"cached": 0, "skipped": "disabled"}
|
||||
assert plex_on["plex"].asked == []
|
||||
|
||||
|
||||
def test_plex_not_answering_ends_the_portrait_run(db, cache_dir, plex_on):
|
||||
_faces(db, 8)
|
||||
plex_on["plex"] = _Plex(down=True)
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 0, "failed": 3}
|
||||
assert len(plex_on["plex"].asked) == 3
|
||||
assert not list(cache_dir.glob("*.miss")) # Plex was down, the files are not to blame
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [401, 403, 429, 500, 503])
|
||||
def test_plex_refusing_is_plex_failing_not_a_dead_file(db, cache_dir, plex_on, code):
|
||||
_faces(db, 8)
|
||||
plex_on["plex"] = _Plex(status=code) # a rotated token, a busy or broken transcoder
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 0, "failed": 3}
|
||||
assert len(plex_on["plex"].asked) == 3
|
||||
assert not list(cache_dir.glob("*.miss")) # the files are not to blame: asked again next run
|
||||
|
||||
|
||||
def test_plex_failing_between_successes_does_not_end_the_run(db, cache_dir, plex_on):
|
||||
_faces(db, 8)
|
||||
plex_on["plex"] = _Plex(down={CDN + f"{i}.jpg" for i in (0, 2, 4, 6)}) # down, ok, down, ok, …
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 4, "failed": 4}
|
||||
|
||||
|
||||
def test_plex_failing_between_dead_files_does_not_end_the_run(db, cache_dir, plex_on):
|
||||
_faces(db, 8)
|
||||
plex_on["plex"] = _Plex( # down, dead, down, dead, …: Plex answers in between
|
||||
down={CDN + f"{i}.jpg" for i in (0, 2, 4, 6)}, dead={CDN + f"{i}.jpg" for i in (1, 3, 5, 7)}
|
||||
)
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 0, "failed": 8}
|
||||
assert len(plex_on["plex"].asked) == 8
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gone", [404, 410])
|
||||
def test_dead_headshots_are_left_alone_and_the_rest_still_land(db, cache_dir, plex_on, gone):
|
||||
_faces(db, 8)
|
||||
dead = {CDN + f"{i}.jpg" for i in range(6)} # more dead files than any failure limit
|
||||
plex_on["plex"] = _Plex(dead=dead, gone=gone)
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 2, "failed": 6}
|
||||
|
||||
plex_on["plex"] = again = _Plex(dead=dead, gone=gone)
|
||||
assert plex_people.warm_portraits(db, 100) == {"cached": 0}
|
||||
assert again.asked == [] # dead ones marked, live ones cached: nothing to ask for
|
||||
|
||||
|
||||
def test_the_person_image_route(db, cache_dir, monkeypatch):
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.state as state
|
||||
from app.main import app
|
||||
from app.routes.plex import playback
|
||||
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
monkeypatch.setattr(playback, "_require_session", lambda request: 1)
|
||||
client = TestClient(app)
|
||||
url = CDN + "face.jpg"
|
||||
small, original = plex_people._portrait_keys(url)
|
||||
|
||||
def get(u):
|
||||
return client.get("/api/plex/person-image", params={"u": u})
|
||||
|
||||
# Only a headshot-shaped url on the CDN host: no query variants of one file, no spellings an HTTP
|
||||
# client folds onto it, no other host.
|
||||
for bad in (
|
||||
url + "?n=1",
|
||||
url + "#x",
|
||||
CDN + "x/../face.jpg",
|
||||
CDN + "./face.jpg",
|
||||
plex_people.PERSON_IMG_HOST + "people//face.jpg",
|
||||
CDN + ".face.jpg",
|
||||
"https://example.com/people/a.jpg",
|
||||
):
|
||||
assert get(bad).status_code == 400, bad
|
||||
assert plex_people.person_thumb_url(bad) is None, bad # never stored either
|
||||
|
||||
# No Plex: the original from the CDN, kept under its own key (the job resizes it later).
|
||||
def not_configured(d):
|
||||
raise plex_people.PlexNotConfigured()
|
||||
|
||||
class _Cdn:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def get(self, u):
|
||||
return httpx.Response(
|
||||
200,
|
||||
content=b"original",
|
||||
headers={"content-type": "image/jpeg"},
|
||||
request=httpx.Request("GET", u),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(playback, "PlexClient", not_configured)
|
||||
monkeypatch.setattr(plex_people.httpx, "Client", _Cdn)
|
||||
resp = get(url)
|
||||
assert (resp.status_code, resp.content) == (200, b"original")
|
||||
assert imgcache.has(original) and not imgcache.has(small)
|
||||
monkeypatch.setattr(plex_people.httpx, "Client", None) # the CDN is not asked again
|
||||
assert get(url).content == b"original" # the kept original answers
|
||||
|
||||
# With Plex: resized, and it replaces the original.
|
||||
monkeypatch.setattr(playback, "PlexClient", lambda d: _Plex())
|
||||
imgcache.drop(original)
|
||||
resp = get(url)
|
||||
assert (resp.status_code, resp.content) == (200, b"small")
|
||||
assert imgcache.has(small)
|
||||
assert get(url).content == b"small" # the cache answers now
|
||||
|
||||
monkeypatch.setattr(playback, "PlexClient", lambda d: _Plex(down=True))
|
||||
assert get(CDN + "other.jpg").status_code == 502
|
||||
|
||||
|
||||
def test_the_people_job_runs_its_halves_without_a_tmdb_key(db, cache_dir, plex_on, monkeypatch):
|
||||
_faces(db, 1)
|
||||
monkeypatch.setattr(plex_tmdb.sysconfig, "get_str", lambda d, k: "")
|
||||
assert plex_tmdb.run_people_enrich(db) == {
|
||||
"bios": {"skipped": "tmdb_api_key not set"},
|
||||
"portraits": {"cached": 1},
|
||||
}
|
||||
|
||||
|
||||
def test_a_phase_count_is_a_change_worth_an_audit_row():
|
||||
assert scheduler._run_changed({"bios": {"enriched": 3}, "portraits": {"cached": 0}})
|
||||
assert scheduler._run_changed({"recent": {"videos_added": 2}, "deep": {}})
|
||||
assert not scheduler._run_changed({"bios": {"skipped": "x"}, "portraits": {"cached": 0}})
|
||||
|
||||
|
||||
def test_the_person_page_says_a_bio_is_on_its_way(db, monkeypatch, override_dep):
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.routes.plex import people
|
||||
|
||||
db.add_all(
|
||||
[
|
||||
PlexPerson(name="Pending", tmdb_person_id=1),
|
||||
PlexPerson(name="Known", tmdb_person_id=2, tmdb_enriched_at=DONE, biography="Bio."),
|
||||
PlexPerson(name="Nobody"),
|
||||
]
|
||||
)
|
||||
user = User(email="enriching@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
override_dep(get_db, db.info["request"])
|
||||
override_dep(current_user, lambda: user)
|
||||
monkeypatch.setattr(people, "_enrich_person_bg", lambda pid: None)
|
||||
key = {"value": "key"}
|
||||
monkeypatch.setattr(people.sysconfig, "get_str", lambda d, k: key["value"])
|
||||
|
||||
def enriching(name):
|
||||
resp = TestClient(app).get(f"/api/plex/person/{name}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["enriching"]
|
||||
|
||||
assert enriching("Pending") is True
|
||||
assert enriching("Known") is False
|
||||
assert enriching("Nobody") is False
|
||||
key["value"] = "" # no TMDB key: nothing will ever arrive, so the page must not wait
|
||||
assert enriching("Pending") is False
|
||||
|
||||
|
||||
def test_known_for_is_the_most_voted_roles_not_guest_visits():
|
||||
payload = {
|
||||
"combined_credits": {
|
||||
"cast": [
|
||||
# Popular this week, but a visit as himself: not what he is known for.
|
||||
{
|
||||
"id": 1,
|
||||
"media_type": "tv",
|
||||
"name": "Late Show",
|
||||
"character": "Himself - Guest",
|
||||
"popularity": 900,
|
||||
"vote_count": 50,
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"media_type": "tv",
|
||||
"name": "Talk Hour",
|
||||
"character": "",
|
||||
"genre_ids": [10767],
|
||||
"popularity": 800,
|
||||
"vote_count": 400,
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"media_type": "movie",
|
||||
"title": "Small Film",
|
||||
"character": "Bob",
|
||||
"popularity": 90,
|
||||
"vote_count": 300,
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"media_type": "movie",
|
||||
"title": "Big Film",
|
||||
"character": "Hero",
|
||||
"popularity": 10,
|
||||
"vote_count": 9000,
|
||||
},
|
||||
],
|
||||
"crew": [
|
||||
{
|
||||
"id": 5,
|
||||
"media_type": "movie",
|
||||
"title": "His Own Film",
|
||||
"job": "Director",
|
||||
"vote_count": 1000,
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"media_type": "movie",
|
||||
"title": "Thanked In",
|
||||
"job": "Thanks",
|
||||
"vote_count": 99999,
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
kf = plex_tmdb.parse_person(payload)["known_for"]
|
||||
assert [k["title"] for k in kf] == ["Big Film", "His Own Film", "Small Film"]
|
||||
|
||||
|
||||
def test_the_page_shows_at_most_fifteen_known_for_titles(db):
|
||||
from app.routes.plex import people
|
||||
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
for i in range(1, 21): # TMDB ids start at 1
|
||||
db.add(
|
||||
PlexItem(
|
||||
rating_key=f"k{i}", library_id=lib.id, kind="movie", title=f"T{i}", guid=f"tmdb:{i}"
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
pool = [{"tmdb_id": i, "media_type": "movie", "title": f"T{i}"} for i in range(1, 21)]
|
||||
cards = people._resolve_known_for(db, pool, 0, [lib.id], [], "en")
|
||||
assert [c["title"] for c in cards] == [f"T{i}" for i in range(1, 16)]
|
||||
@@ -133,6 +133,7 @@ def test_person_detail_unknown_name_is_empty(db, monkeypatch, override_dep):
|
||||
"place_of_birth": None,
|
||||
"known_for_department": None,
|
||||
"also_known_as": [],
|
||||
"enriching": False,
|
||||
"known_for": [],
|
||||
"movies": [],
|
||||
"shows": [],
|
||||
|
||||
@@ -191,11 +191,12 @@ def test_person_detail_schedules_enrich_off_request_path(db, monkeypatch, overri
|
||||
|
||||
scheduled: list[int] = []
|
||||
monkeypatch.setattr(people, "_enrich_person_bg", lambda pid: scheduled.append(pid))
|
||||
monkeypatch.setattr(people.sysconfig, "get_str", lambda d, k: "key")
|
||||
people._enrich_inflight.clear() # ensure the per-person cooldown guard doesn't suppress this open
|
||||
|
||||
resp = _get(db, user, monkeypatch, override_dep, "/api/plex/person/Lazy P")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["biography"] is None # not enriched yet — the bio lands by the next open
|
||||
assert resp.json()["biography"] is None # not enriched yet — the page asks again (`enriching`)
|
||||
assert len(scheduled) == 1 # the enrich was scheduled off the request path
|
||||
|
||||
|
||||
@@ -332,3 +333,74 @@ def test_person_detail_serves_bio_and_resolves_known_for(db, monkeypatch, overri
|
||||
assert j["also_known_as"] == ["Alt Name"]
|
||||
# Only the in-library known_for credit resolves to a card; the out-of-library one is dropped.
|
||||
assert [c["title"] for c in j["known_for"]] == ["Known Movie"]
|
||||
|
||||
|
||||
# --- E30 S4: the biography in the reader's language --------------------------------------------------
|
||||
# The same /person request appends `translations`; the Hungarian one is kept beside the instance-language
|
||||
# biography, and a Hungarian reader is served it, falling back to the English one when TMDB has none.
|
||||
|
||||
|
||||
def _translation(lang: str, bio: str) -> dict:
|
||||
return {"iso_639_1": lang, "iso_3166_1": lang.upper(), "data": {"biography": bio}}
|
||||
|
||||
|
||||
def test_parse_person_keeps_the_hungarian_biography():
|
||||
payload = {
|
||||
**PAYLOAD,
|
||||
"translations": {
|
||||
"translations": [
|
||||
_translation("de", "Ein Schauspieler."),
|
||||
_translation("hu", " Ünnepelt színész. "),
|
||||
]
|
||||
},
|
||||
}
|
||||
assert plex_tmdb.parse_person(payload)["biography_hu"] == "Ünnepelt színész."
|
||||
|
||||
|
||||
def test_parse_person_without_a_hungarian_biography():
|
||||
assert plex_tmdb.parse_person(PAYLOAD)["biography_hu"] is None
|
||||
blank = {**PAYLOAD, "translations": {"translations": [_translation("hu", " ")]}}
|
||||
assert plex_tmdb.parse_person(blank)["biography_hu"] is None
|
||||
|
||||
|
||||
def test_fetch_person_asks_for_the_translations():
|
||||
client = plex_tmdb.TmdbClient.__new__(plex_tmdb.TmdbClient)
|
||||
seen: list[dict] = []
|
||||
client._get = lambda path, params=None: seen.append(params or {}) or {} # type: ignore[method-assign]
|
||||
client.fetch_person(500)
|
||||
assert "translations" in seen[0]["append_to_response"].split(",")
|
||||
|
||||
|
||||
def test_person_detail_serves_the_biography_in_the_readers_language(db, monkeypatch, override_dep):
|
||||
from datetime import datetime, timezone
|
||||
|
||||
done = datetime(2024, 1, 1, tzinfo=timezone.utc)
|
||||
db.add(PlexLibrary(plex_key="7", title="Movies", kind="movie", enabled=True))
|
||||
db.add_all(
|
||||
[
|
||||
PlexPerson(
|
||||
name="Both P",
|
||||
tmdb_person_id=1,
|
||||
tmdb_enriched_at=done,
|
||||
biography="English bio.",
|
||||
biography_hu="Magyar életrajz.",
|
||||
),
|
||||
PlexPerson(
|
||||
name="Only EN P", tmdb_person_id=2, tmdb_enriched_at=done, biography="Only English."
|
||||
),
|
||||
]
|
||||
)
|
||||
user = User(email="biolang@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
|
||||
def bio(name: str, lang: str) -> str | None:
|
||||
resp = _get(db, user, monkeypatch, override_dep, f"/api/plex/person/{name}?lang={lang}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()["biography"]
|
||||
|
||||
assert bio("Both P", "hu") == "Magyar életrajz."
|
||||
assert bio("Both P", "en") == "English bio."
|
||||
assert (
|
||||
bio("Only EN P", "hu") == "Only English."
|
||||
) # TMDB has no Hungarian one: the English stands in
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
"""E28 S3e2 — putting removed Plex playlist rows back WHERE THEY WERE: the list page's "Undo".
|
||||
|
||||
A removal can take one row, a season or a whole show, and a season's episodes need not sit together
|
||||
(a reorder inside the show interleaves them), so each key carries its own index — over the ORDERED
|
||||
rows, since positions have gaps after a removal. One request: an append plus a second reorder call
|
||||
leaves the rows at the end whenever that call fails, and would overwrite another tab's changes.
|
||||
"""
|
||||
|
||||
from app.models import PlexItem, PlexLibrary, PlexPlaylist, PlexPlaylistItem
|
||||
|
||||
|
||||
def _seed(db, user, rks, *, positions=None, extra=("x", "y")):
|
||||
lib = PlexLibrary(plex_key="p1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
items = {}
|
||||
for rk in [*rks, *extra]:
|
||||
it = PlexItem(rating_key=rk, library_id=lib.id, kind="movie", title=rk.upper())
|
||||
db.add(it)
|
||||
items[rk] = it
|
||||
pl = PlexPlaylist(user_id=user.id, title="L")
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for i, rk in enumerate(rks):
|
||||
pos = (positions or range(99))[i]
|
||||
db.add(PlexPlaylistItem(playlist_id=pl.id, item_id=items[rk].id, position=pos))
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
def _order(db, pid):
|
||||
db.expire_all()
|
||||
rows = db.execute(
|
||||
PlexPlaylistItem.__table__.select()
|
||||
.where(PlexPlaylistItem.playlist_id == pid)
|
||||
.order_by(PlexPlaylistItem.position, PlexPlaylistItem.id)
|
||||
).all()
|
||||
by_id = {it.id: it.rating_key for it in db.query(PlexItem).all()}
|
||||
return [by_id[r.item_id] for r in rows], [r.position for r in rows]
|
||||
|
||||
|
||||
def _add(client, pid, keys, positions=None):
|
||||
body = {"item_rating_keys": keys}
|
||||
if positions is not None:
|
||||
body["positions"] = positions
|
||||
return client.post(f"/api/plex/playlists/{pid}/items/bulk", json=body)
|
||||
|
||||
|
||||
def test_one_row_goes_back_to_its_index(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"])
|
||||
r = _add(as_user(user), pl.id, ["x"], [1])
|
||||
assert r.status_code == 200
|
||||
assert r.json()["added"] == 1
|
||||
assert _order(db, pl.id) == (["a", "x", "b", "c"], [0, 1, 2, 3])
|
||||
|
||||
|
||||
def test_scattered_rows_go_back_to_each_of_their_indices(db, make_user, as_user):
|
||||
# A season whose episodes were interleaved with another season's: removed rows at 1 and 3 of
|
||||
# a, x, b, y, c. Given in any order, each lands where it stood.
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"])
|
||||
_add(as_user(user), pl.id, ["y", "x"], [3, 1])
|
||||
assert _order(db, pl.id)[0] == ["a", "x", "b", "y", "c"]
|
||||
|
||||
|
||||
def test_indices_count_rows_not_stored_numbers(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"], positions=[0, 7, 20])
|
||||
_add(as_user(user), pl.id, ["x"], [2])
|
||||
assert _order(db, pl.id) == (["a", "b", "x", "c"], [0, 1, 2, 3])
|
||||
|
||||
|
||||
def test_an_index_past_the_end_appends(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a"])
|
||||
_add(as_user(user), pl.id, ["x"], [50])
|
||||
assert _order(db, pl.id)[0] == ["a", "x"]
|
||||
|
||||
|
||||
def test_a_row_already_in_the_list_does_not_move(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c"])
|
||||
r = _add(as_user(user), pl.id, ["c"], [0])
|
||||
assert r.json()["added"] == 0
|
||||
assert _order(db, pl.id)[0] == ["a", "b", "c"]
|
||||
|
||||
|
||||
def test_without_positions_it_still_appends(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b"])
|
||||
_add(as_user(user), pl.id, ["y", "x"])
|
||||
assert _order(db, pl.id)[0] == ["a", "b", "y", "x"]
|
||||
|
||||
|
||||
def test_positions_must_pair_with_the_keys(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a"])
|
||||
assert _add(as_user(user), pl.id, ["x", "y"], [0]).status_code == 422
|
||||
assert _add(as_user(user), pl.id, ["x"], [-1]).status_code == 422
|
||||
assert _order(db, pl.id)[0] == ["a"]
|
||||
|
||||
|
||||
def test_remove_then_put_back_is_no_change(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user, ["a", "b", "c", "d"], extra=())
|
||||
client = as_user(user)
|
||||
client.post(
|
||||
f"/api/plex/playlists/{pl.id}/items/remove-bulk", json={"item_rating_keys": ["b", "d"]}
|
||||
)
|
||||
assert _order(db, pl.id)[0] == ["a", "c"]
|
||||
_add(client, pl.id, ["b", "d"], [1, 3])
|
||||
assert _order(db, pl.id)[0] == ["a", "b", "c", "d"]
|
||||
@@ -0,0 +1,115 @@
|
||||
"""A sidecar subtitle in a legacy code page must render its accents, not U+FFFD.
|
||||
|
||||
Found live on Arrow S03E15 (a Hungarian .srt beside the .mp4): the served WebVTT held 1391 U+FFFD and
|
||||
not one accented letter, because `_sidecar_vtt` decoded the bytes as UTF-8 with `errors="replace"`
|
||||
while the file was Windows-1250 (ő = 0xF5, ű = 0xFB). Legacy-code-page .srt files are the norm for
|
||||
Hungarian releases, and SubRip carries no charset of its own, so the stream's language is the only hint.
|
||||
No Postgres: Plex and ffmpeg are stubbed.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.plex import subtitles
|
||||
|
||||
HU = "1\r\n00:00:01,000 --> 00:00:02,000\r\nMiért nem szóltál? Ősz, űr.\r\n"
|
||||
FR = "1\r\n00:00:01,000 --> 00:00:02,000\r\nDéjà vu, garçon.\r\n"
|
||||
RU = "1\r\n00:00:01,000 --> 00:00:02,000\r\nПривет, мир.\r\n"
|
||||
|
||||
|
||||
class _Plex:
|
||||
def __init__(self, raw: bytes):
|
||||
self.raw = raw
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def raw_get(self, key: str) -> bytes:
|
||||
return self.raw
|
||||
|
||||
|
||||
def _sidecar(monkeypatch, raw: bytes, lang: str | None, codec: str = "srt") -> str:
|
||||
monkeypatch.setattr(subtitles, "PlexClient", lambda db: _Plex(raw))
|
||||
stream = {"key": "/library/streams/1", "codec": codec, "languageCode": lang}
|
||||
return subtitles._sidecar_vtt(cast(Session, None), stream)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("text", "encoding", "lang"),
|
||||
[
|
||||
(HU, "cp1250", "hun"), # the file that was found broken
|
||||
(FR, "cp1252", "fre"), # a Western legacy file
|
||||
(RU, "cp1251", "rus"), # a Cyrillic one
|
||||
(HU, "utf-8", "hun"), # already UTF-8: untouched
|
||||
(HU, "utf-8-sig", "hun"), # UTF-8 with a BOM
|
||||
(HU, "utf-16", "hun"), # UTF-16 (carries its BOM)
|
||||
],
|
||||
ids=["cp1250-hu", "cp1252-fr", "cp1251-ru", "utf8", "utf8-bom", "utf16"],
|
||||
)
|
||||
def test_a_sidecar_srt_is_decoded_in_its_own_encoding(monkeypatch, text, encoding, lang):
|
||||
vtt = _sidecar(monkeypatch, text.encode(encoding), lang)
|
||||
assert "\ufffd" not in vtt
|
||||
assert vtt.startswith("WEBVTT\n\n")
|
||||
assert text.splitlines()[2] in vtt
|
||||
|
||||
|
||||
def test_a_non_srt_sidecar_reaches_ffmpeg_as_utf8(monkeypatch):
|
||||
seen: dict[str, bytes] = {}
|
||||
|
||||
def fake_ffmpeg(src: str, sub_ord):
|
||||
seen["bytes"] = Path(src).read_bytes()
|
||||
return "WEBVTT\n"
|
||||
|
||||
monkeypatch.setattr(subtitles, "ffmpeg_to_vtt", fake_ffmpeg)
|
||||
_sidecar(monkeypatch, "[Events]\nDialogue: Ősz\n".encode("cp1250"), "hun", codec="ass")
|
||||
assert seen["bytes"].decode("utf-8") == "[Events]\nDialogue: Ősz\n"
|
||||
|
||||
|
||||
def test_a_track_cached_before_the_encoding_fix_is_not_served(tmp_path, monkeypatch):
|
||||
"""The cache has no eviction, so a mis-decoded track would have stayed broken for good: the key
|
||||
is versioned, and a track cached under the old one is read again (once) instead of served."""
|
||||
monkeypatch.setattr(subtitles, "CACHE_DIR", tmp_path)
|
||||
(tmp_path / "sub_14881_1.vtt").write_text(
|
||||
"WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAz�rt\n", "utf-8"
|
||||
)
|
||||
assert subtitles.cached_track("14881", 1)[0] is None
|
||||
|
||||
|
||||
def test_one_stray_byte_does_not_send_a_utf8_file_to_a_legacy_code_page(monkeypatch):
|
||||
"""A UTF-8 release with ONE bad byte (a cut-and-paste Windows-1252 quote) failed strict UTF-8 as a
|
||||
whole, and the whole file was then read as Windows-1250: every correct ő/é became mojibake. Nearly
|
||||
all of it is valid UTF-8, so it stays UTF-8 and only the one byte is replaced."""
|
||||
raw = (HU * 20).encode("utf-8") + b"\x93quoted\x94\r\n"
|
||||
vtt = _sidecar(monkeypatch, raw, "hun")
|
||||
assert "Miért nem szóltál? Ősz, űr." in vtt
|
||||
assert "Miért" not in vtt
|
||||
|
||||
|
||||
def test_an_untagged_legacy_sidecar_is_read_as_central_european(monkeypatch):
|
||||
"""No language tag: the fallback was Windows-1252, which turns a Hungarian ő/ű into õ/û."""
|
||||
vtt = _sidecar(monkeypatch, HU.encode("cp1250"), None)
|
||||
assert "Ősz, űr." in vtt
|
||||
|
||||
|
||||
def test_a_latin_serbian_sidecar_is_read_as_central_european(monkeypatch):
|
||||
"""Serbian releases are overwhelmingly Latin script in Windows-1250 (like hrv/bos); read as
|
||||
Windows-1251 every č/ć/š/ž/đ turned into Cyrillic noise."""
|
||||
sr = "1\r\n00:00:01,000 --> 00:00:02,000\r\nŠta ćeš, đače? Čekaj.\r\n"
|
||||
vtt = _sidecar(monkeypatch, sr.encode("cp1250"), "srp")
|
||||
assert "Šta ćeš, đače? Čekaj." in vtt
|
||||
|
||||
|
||||
def test_a_track_cached_under_the_previous_decoder_is_not_served(tmp_path, monkeypatch):
|
||||
"""The decoder changed again (stray-byte UTF-8, cp1250 fallbacks): the key is versioned once more
|
||||
so a track the old rules mis-decoded is read again instead of served for good."""
|
||||
monkeypatch.setattr(subtitles, "CACHE_DIR", tmp_path)
|
||||
(tmp_path / "sub2_14881_1.vtt").write_text(
|
||||
"WEBVTT\n\n1\n00:00:01.000 --> 00:00:02.000\nAzõrt\n", "utf-8"
|
||||
)
|
||||
assert subtitles.cached_track("14881", 1)[0] is None
|
||||
@@ -311,8 +311,9 @@ def test_covered_ranges_coalesce_and_answer_membership():
|
||||
def test_a_cached_vtt_from_an_older_build_counts_as_complete(tmp_path):
|
||||
"""Before this change the cache only ever held whole tracks, and none of them has a sidecar.
|
||||
Reading one as "covers nothing" would re-extract every already-cached track after the upgrade."""
|
||||
(tmp_path / "sub_98614_0.vtt").write_text(_vtt(_cue("00:01.000", "00:02.000", "x")), "utf-8")
|
||||
assert subtitles.read_meta("sub_98614_0") == ([[0.0, 1_000_000_000.0]], True)
|
||||
key = subtitles.cache_key("98614", 0)
|
||||
(tmp_path / f"{key}.vtt").write_text(_vtt(_cue("00:01.000", "00:02.000", "x")), "utf-8")
|
||||
assert subtitles.read_meta(key) == ([[0.0, 1_000_000_000.0]], True)
|
||||
|
||||
|
||||
# --- the deep-review round: the seams between the window and the full pass ----------------------
|
||||
|
||||
@@ -49,7 +49,7 @@ MOVIE_PAYLOAD = {
|
||||
],
|
||||
},
|
||||
"production_companies": [{"name": "Paramount"}, {"name": "Alfran"}],
|
||||
"production_countries": [{"name": "United States of America"}],
|
||||
"production_countries": [{"iso_3166_1": "US", "name": "United States of America"}],
|
||||
"recommendations": {
|
||||
"results": [
|
||||
{
|
||||
@@ -126,7 +126,8 @@ def test_parse_movie_pulls_all_fields():
|
||||
},
|
||||
] # the Gaffer is filtered; the two Coppola jobs survive (dedup is by name+job)
|
||||
assert d["studios"] == ["Paramount", "Alfran"]
|
||||
assert d["production_countries"] == ["United States of America"]
|
||||
# E30: the code beside the English name, so the client can name the country in the reader's language.
|
||||
assert d["production_countries"] == [{"code": "US", "name": "United States of America"}]
|
||||
assert [r["tmdb_id"] for r in d["recommendations"]] == [240, 999]
|
||||
assert d["backdrops"] == ["/bd1.jpg", "/bd2.jpg"]
|
||||
# cast + crew names → tmdb person ids for the plex_people foundation
|
||||
@@ -255,13 +256,13 @@ def test_serve_block_none_until_enriched_and_resolves_recs(db):
|
||||
db.add_all([sequel, fresh])
|
||||
db.commit()
|
||||
|
||||
assert tmdb.serve_block(db, fresh) is None # not enriched yet
|
||||
assert tmdb.serve_block(db, fresh, lang="en") is None # not enriched yet
|
||||
|
||||
tmdb.apply_tmdb_detail(fresh, tmdb.parse_tmdb(MOVIE_PAYLOAD, "movie"))
|
||||
fresh.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
|
||||
block = tmdb.serve_block(db, fresh)
|
||||
block = tmdb.serve_block(db, fresh, lang="en")
|
||||
assert block is not None
|
||||
assert block["budget"] == 6000000
|
||||
assert block["keywords"] == ["mafia", "crime family"]
|
||||
@@ -409,6 +410,7 @@ def test_resolve_recs_matches_by_media_type_not_bare_id(db):
|
||||
{"tmdb_id": 500, "title": "Movie 500", "media_type": "movie"},
|
||||
{"tmdb_id": 900, "title": "Show 900", "media_type": "tv"},
|
||||
],
|
||||
lang="en",
|
||||
)
|
||||
}
|
||||
assert miss[500]["in_library"] is False and miss[500]["rating_key"] is None
|
||||
@@ -423,7 +425,74 @@ def test_resolve_recs_matches_by_media_type_not_bare_id(db):
|
||||
{"tmdb_id": 900, "media_type": "movie"},
|
||||
{"tmdb_id": 500, "media_type": "tv"},
|
||||
],
|
||||
lang="en",
|
||||
)
|
||||
}
|
||||
assert hit[900]["in_library"] is True and hit[900]["rating_key"] == "m900"
|
||||
assert hit[500]["in_library"] is True and hit[500]["rating_key"] == "s500"
|
||||
|
||||
|
||||
# ---- E30: countries by ISO code -----------------------------------------------------------------
|
||||
|
||||
|
||||
class FakeCountries(FakeTmdb):
|
||||
"""TMDB's /configuration/countries, counting the requests."""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
def countries(self):
|
||||
self.calls += 1
|
||||
return [
|
||||
{"iso_3166_1": "US", "english_name": "United States of America"},
|
||||
{"iso_3166_1": "YU", "english_name": "Yugoslavia"},
|
||||
]
|
||||
|
||||
|
||||
def test_countries_stored_before_codes_get_them_from_tmdbs_list(db):
|
||||
lib = _lib(db)
|
||||
old = PlexItem(
|
||||
rating_key="m1",
|
||||
library_id=lib.id,
|
||||
kind="movie",
|
||||
title="Old",
|
||||
production_countries=["Yugoslavia", "Atlantis"],
|
||||
tmdb_enriched_at=datetime.now(timezone.utc),
|
||||
)
|
||||
show_lib = _lib(db, key="2", kind="show")
|
||||
show = PlexShow(
|
||||
rating_key="s1",
|
||||
library_id=show_lib.id,
|
||||
title="Show",
|
||||
production_countries=["United States of America"],
|
||||
)
|
||||
new = PlexItem(
|
||||
rating_key="m2",
|
||||
library_id=lib.id,
|
||||
kind="movie",
|
||||
title="New",
|
||||
production_countries=[{"code": "US", "name": "United States of America"}],
|
||||
)
|
||||
db.add_all([old, show, new])
|
||||
db.commit()
|
||||
|
||||
# Served before the backfill: the bare name, with no code — never a crash on the old shape.
|
||||
assert tmdb.serve_block(db, old, lang="hu")["production_countries"] == [
|
||||
{"code": None, "name": "Yugoslavia"},
|
||||
{"code": None, "name": "Atlantis"},
|
||||
]
|
||||
|
||||
fake = FakeCountries()
|
||||
assert tmdb.backfill_country_codes(db, fake) == 2
|
||||
db.refresh(old)
|
||||
db.refresh(show)
|
||||
assert old.production_countries == [
|
||||
{"code": "YU", "name": "Yugoslavia"},
|
||||
{"code": None, "name": "Atlantis"}, # not in TMDB's list → no code, the name kept
|
||||
]
|
||||
assert show.production_countries == [{"code": "US", "name": "United States of America"}]
|
||||
|
||||
# Nothing left to code: no request at all.
|
||||
assert tmdb.backfill_country_codes(db, fake) == 0
|
||||
assert fake.calls == 1
|
||||
|
||||
@@ -18,12 +18,12 @@ _PW = "open sesame"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unlock_url(as_user, make_user, tmp_path, monkeypatch):
|
||||
def unlock_url(as_user, make_user, tmp_path, monkeypatch, upload_headers):
|
||||
monkeypatch.setattr(settings, "download_root", str(tmp_path))
|
||||
monkeypatch.setattr(probemod, "probe_media", lambda _p: ("h264", "aac", 320, 240, 5))
|
||||
owner = as_user(make_user())
|
||||
job = owner.post(
|
||||
"/api/downloads/upload", files={"file": ("clip.mp4", b"fake-video", "video/mp4")}
|
||||
"/api/downloads/upload", content=b"fake-video", headers=upload_headers("clip.mp4")
|
||||
)
|
||||
assert job.status_code == 200, job.text
|
||||
link = owner.post(f"/api/downloads/{job.json()['id']}/links", json={"password": _PW})
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
"""The Python major is a contract, not a coincidence — the backend half (E27 S3).
|
||||
|
||||
`0.93.2` was released to fix exactly this bug on the JavaScript side: four machines built or checked
|
||||
the app and none of them agreed on which Node they ran, the server gate went red for 73 of its last
|
||||
90 runs, and nothing told anyone. The Python side had the same shape and nobody had looked: prod
|
||||
(`Dockerfile`), the test image (`Dockerfile.test`) and CI (`check.yml`) each spelled **3.12** as an
|
||||
independent literal, there was no `.python-version`, no `requires-python`, and ruff had no
|
||||
`target-version` — while the dev machine had no 3.12 installed at all, only 3.13 and 3.14.
|
||||
|
||||
**The contract is a chain, and it is split across two test files because no single runner can see
|
||||
every link.** This suite runs inside the test image, which mounts `backend/` and nothing else, so it
|
||||
cannot read the repo-root Dockerfile or the workflow. It therefore holds the link it CAN see, and the
|
||||
one that matters most here:
|
||||
|
||||
the interpreter actually running == backend/pyrightconfig.json == backend/ruff.toml
|
||||
|
||||
and `frontend/src/lib/pythonEngine.contract.test.ts` — which runs on the host, with the whole tree —
|
||||
holds the rest of the chain:
|
||||
|
||||
/.python-version == pyrightconfig.json == ruff.toml == Dockerfile == Dockerfile.test == check.yml
|
||||
|
||||
Together they close the loop. To move the version: edit `/.python-version`, then follow the failures
|
||||
in both files. Every place they name is a place that would otherwise have been forgotten.
|
||||
"""
|
||||
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
BACKEND = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def _version_in(path: Path, pattern: str) -> tuple[int, int]:
|
||||
"""The (major, minor) the given file declares. A rename or a moved line lands here — re-point
|
||||
the test, the contract did not go away with it."""
|
||||
text = path.read_text(encoding="utf-8")
|
||||
match = re.search(pattern, text, re.MULTILINE)
|
||||
assert match is not None, f"{pattern!r} not found in {path.name}"
|
||||
return int(match.group(1)), int(match.group(2))
|
||||
|
||||
|
||||
# pyrightconfig.json is JSONC (it carries comments), so it is read by regex rather than json.loads.
|
||||
PINNED = _version_in(BACKEND / "pyrightconfig.json", r'"pythonVersion"\s*:\s*"(\d+)\.(\d+)"')
|
||||
|
||||
|
||||
def test_pins_a_plausible_python_version():
|
||||
"""Guards against a typo agreeing with nothing at all."""
|
||||
assert PINNED >= (3, 12), PINNED
|
||||
|
||||
|
||||
def test_this_interpreter_is_the_pinned_one():
|
||||
"""The assertion that pays. A green suite on another minor does not mean the shipped image is
|
||||
green — it means the suite and the image have drifted and nobody noticed yet."""
|
||||
running = sys.version_info[:2]
|
||||
assert running == PINNED, (
|
||||
f"these tests are running on Python {running[0]}.{running[1]}, but the app is pinned to "
|
||||
f"{PINNED[0]}.{PINNED[1]} (backend/pyrightconfig.json, /.python-version). The suite runs in "
|
||||
f"the test image, so this failing means backend/Dockerfile.test has drifted from the pin."
|
||||
)
|
||||
|
||||
|
||||
def test_ruff_judges_the_pinned_version():
|
||||
"""Unset, ruff falls back to a default that has nothing to do with us, and its syntax and
|
||||
upgrade advice are then aimed at some other interpreter."""
|
||||
major, minor = _version_in(BACKEND / "ruff.toml", r'^target-version\s*=\s*"py(\d)(\d+)"')
|
||||
assert (major, minor) == PINNED
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user