Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0528b35f2d | ||
|
|
cb46774851 | ||
|
|
f1cc6ce5dc | ||
|
|
4efe799eda | ||
|
|
40b27f2982 | ||
|
|
0314c4cce2 | ||
|
|
182ec31120 | ||
|
|
12a39f5291 | ||
|
|
79704b0001 | ||
|
|
4191189b4c | ||
|
|
7dbda60815 | ||
|
|
d1df76e8e5 | ||
|
|
3f6a9efd57 | ||
|
|
64e8f1da3d | ||
|
|
a1224e9a8c | ||
|
|
da4b19a39a | ||
|
|
5a74cf86b7 | ||
|
|
e432dc2780 | ||
|
|
3741d1a46d | ||
|
|
98ec1abf25 | ||
|
|
e5b155ada9 | ||
|
|
e09d613b04 | ||
|
|
38943fa93e | ||
|
|
887b4e546c | ||
|
|
df4a79a1f1 | ||
|
|
5d10a521a8 | ||
|
|
d7fb7075b8 | ||
|
|
a486c7521a | ||
|
|
76358a0d77 | ||
|
|
dc00b55feb | ||
|
|
fb6b56f121 | ||
|
|
2e2d6f5746 | ||
|
|
2ab9ff6a42 | ||
|
|
378c4daf0f | ||
|
|
8f1208fdfe | ||
|
|
52f12e5331 | ||
|
|
c4abbe48c1 | ||
|
|
4fc307c426 | ||
|
|
328d415cfe | ||
|
|
43e4929b45 | ||
|
|
5806adda7e | ||
|
|
c73761ae39 | ||
|
|
ca9ca85b18 | ||
|
|
614d728561 | ||
|
|
783f6408f9 | ||
|
|
751e1ae5ed | ||
|
|
da554df7c7 | ||
|
|
b1ff4058ac | ||
|
|
7ed2ac4e95 | ||
|
|
cff8fe9a6a | ||
|
|
471846d784 | ||
|
|
3c16012524 | ||
|
|
cdfe7a8986 | ||
|
|
3e96211db0 | ||
|
|
e9c15baae0 | ||
|
|
6edb22988c | ||
|
|
f9c3714025 | ||
|
|
a4775e1cd0 | ||
|
|
b40440b825 | ||
|
|
c4c442bc5b | ||
|
|
c74ceb73e3 | ||
|
|
e0e28c006c | ||
|
|
f8f22fc985 | ||
|
|
90e0baa870 | ||
|
|
dfd43ae89c | ||
|
|
608fde2d55 | ||
|
|
0302c466e8 | ||
|
|
a50055577a | ||
|
|
e16cdbd16f | ||
|
|
15ec67b029 | ||
|
|
ac53a3b757 | ||
|
|
e6de18b8c0 | ||
|
|
fb2e3d8490 | ||
|
|
d928d50686 | ||
|
|
540822d2b0 | ||
|
|
b4ef07d28e | ||
|
|
e995c664c6 | ||
|
|
20010178b0 | ||
|
|
78fdcda6bb | ||
|
|
98d0ca6eeb | ||
|
|
e4c80fb7bb | ||
|
|
be156fc52b | ||
|
|
25905aa786 | ||
|
|
87d71e3762 | ||
|
|
ab146211fa | ||
|
|
ec22eefd31 | ||
|
|
2543d44860 | ||
|
|
29c0715237 | ||
|
|
6b86f99249 | ||
|
|
3563858fd1 | ||
|
|
08e62bbb16 | ||
|
|
9454717f47 | ||
|
|
6ac9ffd45e | ||
|
|
fa55e01ef0 | ||
|
|
ab963c81d2 | ||
|
|
59e6cc027d | ||
|
|
ca94a716d8 | ||
|
|
63d01babe5 | ||
|
|
705377c797 | ||
|
|
2819049133 | ||
|
|
95b94a1d72 | ||
|
|
12077fbebc | ||
|
|
519fbe2a18 | ||
|
|
a39504e617 | ||
|
|
aac79ffa53 | ||
|
|
67c9f10153 | ||
|
|
584d082b6b | ||
|
|
f3a85916ec | ||
|
|
f6298d29af | ||
|
|
632e39d458 | ||
|
|
13d95ef70e | ||
|
|
0f9ed7cdbf | ||
|
|
ffcea0cb07 | ||
|
|
a3f8314c9a | ||
|
|
d5ccb1091a | ||
|
|
528774d640 | ||
|
|
3a9ea9e330 | ||
|
|
25f2d9d0e4 | ||
|
|
5ab6778b97 | ||
|
|
26697472cf | ||
|
|
2abd4ffcb6 | ||
|
|
a9ac303d3e | ||
|
|
740602b9bf | ||
|
|
e31a588f19 | ||
|
|
cda626be10 | ||
|
|
b7ce1d467c | ||
|
|
52fb999af6 | ||
|
|
e0453fa07d | ||
|
|
2525a12264 | ||
|
|
8e09dd85f8 | ||
|
|
addb3d1b32 | ||
|
|
86884e368e | ||
|
|
51f881681b | ||
|
|
27483ae1d6 | ||
|
|
595c29e17c | ||
|
|
acf93a1f36 | ||
|
|
84278293df | ||
|
|
dc24d80ed4 | ||
|
|
3810af66da | ||
|
|
eb339195c7 | ||
|
|
5d1065c1bc | ||
|
|
6ffb5bd8c1 | ||
|
|
dddf79ec2f | ||
|
|
5ebb4cb506 | ||
|
|
0b1d905866 | ||
|
|
52c15adaf2 | ||
|
|
3372626e34 | ||
|
|
98bb841a6a | ||
|
|
98bf0a9d66 | ||
|
|
17bb14fb18 | ||
|
|
70f98b352f | ||
|
|
efa4aeeeda | ||
|
|
a7d41eb5f3 | ||
|
|
76eead976f | ||
|
|
922d589b09 | ||
|
|
a975ab627a | ||
|
|
9bedd96774 | ||
|
|
383c85231a | ||
|
|
de62035889 | ||
|
|
b456232d00 | ||
|
|
c9e95ecea6 | ||
|
|
7987bd8a37 | ||
|
|
3411265dc1 | ||
|
|
098b8dbe43 | ||
|
|
f48f16830e | ||
|
|
955c23d321 | ||
|
|
d3e1d8219c | ||
|
|
4480839db0 | ||
|
|
8e25567ff2 | ||
|
|
8f65bd184e | ||
|
|
ae9c248987 | ||
|
|
bd6aaa1e43 | ||
|
|
2c65edfffd | ||
|
|
6c2c01e67b | ||
|
|
ef343dbec0 | ||
|
|
a58a398427 | ||
|
|
d510c8a534 | ||
|
|
dc5206f9d3 | ||
|
|
89e65ca331 | ||
|
|
17916b411a | ||
|
|
3ef2c38eb4 | ||
|
|
4c63d6660d | ||
|
|
a453b77f04 | ||
|
|
8e1004fe6f | ||
|
|
663f30feeb | ||
|
|
6653d81a80 | ||
|
|
21df7c8348 | ||
|
|
97381e65d0 | ||
|
|
f8ced6d819 | ||
|
|
f254588409 | ||
|
|
3e7c79fa69 | ||
|
|
47c12f9405 | ||
|
|
bd0914b7ed | ||
|
|
079cee4e9c | ||
|
|
edf2f22ef9 | ||
|
|
e1ef56cdbc | ||
|
|
4d0abc61d5 | ||
|
|
7c2b22bafa | ||
|
|
5e5c8a42a9 | ||
|
|
2b3094ffc5 | ||
|
|
9c91eb5d22 | ||
|
|
2df60fbf07 | ||
|
|
a2d17d693c | ||
|
|
c4917619ee | ||
|
|
760bdae278 | ||
|
|
1b946113c6 | ||
|
|
b0e51c6f89 | ||
|
|
7455fff401 | ||
|
|
aaf1843b09 | ||
|
|
a4737b92e4 | ||
|
|
75ae810866 | ||
|
|
9d7526ee57 | ||
|
|
fa0b0edd63 | ||
|
|
bfc6d199c0 | ||
|
|
4d6e59b80a | ||
|
|
4646aae320 | ||
|
|
eddeca82d3 | ||
|
|
69edeecc45 | ||
|
|
6088e476a7 | ||
|
|
ce171fa954 | ||
|
|
113da76c36 | ||
|
|
4ef11901db | ||
|
|
29abf609f3 | ||
|
|
e13d93a3ed | ||
|
|
b9850b1710 | ||
|
|
905a9f2856 | ||
|
|
1d8b83c37c | ||
|
|
62dbf35ea3 | ||
|
|
e9ccd9bc98 | ||
|
|
cc8e0648db | ||
|
|
ff80cc96fd | ||
|
|
295c1db5f1 | ||
|
|
0521149cf1 | ||
|
|
b4a6569d0d | ||
|
|
1833f594cb | ||
|
|
1615838f33 | ||
|
|
7fa8f4ebde | ||
|
|
6354454cc9 | ||
|
|
c47c193ad1 | ||
|
|
db10bfbbca | ||
|
|
dac9d89f91 | ||
|
|
81936e9f56 | ||
|
|
86f467bd65 | ||
|
|
7d480e2ad0 | ||
|
|
ad136c3fe8 | ||
|
|
9c4a999e54 | ||
|
|
3f5634a44f | ||
|
|
f4bc60d659 | ||
|
|
7055548260 | ||
|
|
477271907e | ||
|
|
50923bb684 | ||
|
|
d2e40dcffb | ||
|
|
58878958d8 | ||
|
|
b47d22cf3b |
@@ -4,6 +4,7 @@
|
|||||||
*.env.local
|
*.env.local
|
||||||
**/__pycache__/
|
**/__pycache__/
|
||||||
**/*.pyc
|
**/*.pyc
|
||||||
|
**/.coverage
|
||||||
.venv/
|
.venv/
|
||||||
venv/
|
venv/
|
||||||
# Docker patterns are anchored at the context root with NO implicit `**/` (unlike .gitignore, where
|
# Docker patterns are anchored at the context root with NO implicit `**/` (unlike .gitignore, where
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
||||||
# independent of any one machine. Two jobs match the two local lanes:
|
# independent of any one machine. Two jobs match the two local lanes:
|
||||||
# frontend — tsc (3 projects) + eslint + prettier --check + knip + vitest
|
# frontend — tsc (3 projects) + eslint + prettier --check + knip (both modes) + vitest + vite build
|
||||||
# backend — ruff + the full pytest suite (pure-logic + the DB-backed lane) against a real Postgres
|
# + npm audit of the production dependencies
|
||||||
|
# backend — ruff + basedpyright + the full pytest suite (pure-logic + the DB-backed lane) against
|
||||||
|
# a real Postgres + pip-audit of the lock
|
||||||
#
|
#
|
||||||
# They are declared side by side but they DO NOT run in parallel: the lab has a single act_runner
|
# They are declared side by side but they DO NOT run in parallel: the lab has a single act_runner
|
||||||
# with one slot, so they queue. Measured across runs #86-#91 — the backend job starts at the exact
|
# with one slot, so they queue. Measured across runs #86-#91 — the backend job starts at the exact
|
||||||
@@ -56,6 +58,12 @@ jobs:
|
|||||||
- run: npm run format:check
|
- run: npm run format:check
|
||||||
- run: npm run knip
|
- run: npm run knip
|
||||||
- run: npm run test
|
- run: npm run test
|
||||||
|
# The production bundle (E32): before this, `vite build` — and devOnlyGuard, which fails it if
|
||||||
|
# dev-only code reaches the bundle — first ran inside the image build, after main was pushed.
|
||||||
|
- run: npx vite build
|
||||||
|
# What ships to the browser: a known CVE at high or above in a production dependency fails.
|
||||||
|
# (The dev tooling's own advisories are burned down by hand; see docs/gates.md.)
|
||||||
|
- run: npm audit --omit=dev --audit-level=high
|
||||||
|
|
||||||
backend:
|
backend:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -112,8 +120,9 @@ jobs:
|
|||||||
- name: pytest
|
- name: pytest
|
||||||
run: python -m pytest
|
run: python -m pytest
|
||||||
working-directory: backend
|
working-directory: backend
|
||||||
# Dependency CVE audit over the pinned lock. Non-blocking for now (informational) — a real CVE
|
# Dependency CVE audit over the pinned lock. BLOCKING since E32: it ran advisory-only, and when
|
||||||
# shows up in the run without failing the gate; tighten to a hard fail once the noise is known.
|
# it was finally read it had three urllib3 advisories with a fix available (bumped in the same
|
||||||
- name: pip-audit (advisory)
|
# change). An advisory without a fix is answered with `--ignore-vuln <id>` here, a reason and a
|
||||||
|
# date in the comment, and a line in docs/gates.md.
|
||||||
|
- name: pip-audit
|
||||||
run: pip install --no-cache-dir pip-audit && pip-audit -r backend/requirements.lock
|
run: pip install --no-cache-dir pip-audit && pip-audit -r backend/requirements.lock
|
||||||
continue-on-error: true
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ __pycache__/
|
|||||||
venv/
|
venv/
|
||||||
.mypy_cache/
|
.mypy_cache/
|
||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
|
.coverage
|
||||||
.ruff_cache/
|
.ruff_cache/
|
||||||
|
|
||||||
# Node / frontend
|
# Node / frontend
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
# The CVE gate's allowlist (E32). The publish gate scans the built image and the committed tree with
|
||||||
|
# Trivy and fails on any HIGH or CRITICAL finding that has a fix available. A finding that cannot be
|
||||||
|
# fixed yet is let through ONLY by an entry here, and every entry carries both:
|
||||||
|
# statement — why it is acceptable for now (what is exposed, what is not)
|
||||||
|
# expired_at — the date it stops being accepted; after it, the gate fails again on its own
|
||||||
|
# A contract test (frontend/src/lib/gates.contract.test.ts) refuses an entry without either, and
|
||||||
|
# docs/gates.md lists them.
|
||||||
|
#
|
||||||
|
# Example:
|
||||||
|
# vulnerabilities:
|
||||||
|
# - id: CVE-2026-00000
|
||||||
|
# statement: in libfoo's XML path, which nothing in the image calls; no fixed Debian build yet
|
||||||
|
# expired_at: 2026-12-31
|
||||||
|
vulnerabilities: []
|
||||||
|
secrets: []
|
||||||
+855
-1400
File diff suppressed because it is too large
Load Diff
@@ -12,12 +12,15 @@ router-free navigation provider.
|
|||||||
|
|
||||||
## The gate — never merge or ship red
|
## The gate — never merge or ship red
|
||||||
|
|
||||||
The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + typecheck (3 tsc projects) + eslint + prettier
|
The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + the lock check + typecheck (3 tsc projects) +
|
||||||
--check + knip + vitest + pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
eslint + prettier --check + knip (both modes) + vitest + vite build + pytest**, then a coverage report (no threshold). It runs
|
||||||
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
before every `release` merges and on every `publish`, which **refuses to build on failure — there is deliberately no `--skip`
|
||||||
|
flag**; a release also requires the Playwright suite, and the image must pass a Trivy scan before it is pushed. Run it and get it
|
||||||
|
green **before** any merge, release, or handing work to review.
|
||||||
|
|
||||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||||
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` + `ruff format --check backend` and `pytest` (the DB-lane needs Postgres).
|
- Directly, without the dispatcher: the commands in [`docs/development.md`](docs/development.md) § The gate (frontend lanes, `ruff` over `backend scripts`, basedpyright in the test image, and `pytest` — the DB-lane needs Postgres).
|
||||||
|
- **A gate with any real hit is never switched off or narrowed** — not a lint rule, a compiler flag, a lane or a release step. Fix the sites, or suppress each one inline naming its invariant (`-- why`, lint-enforced). What stays narrowed is in [`docs/gates.md`](docs/gates.md), and a contract test fails when a narrowing is missing from it. The effective ESLint config of every file and the CI/lane lines are pinned in `frontend/src/lib/gates.snapshot.json`: a new code file or any gate change fails until you regenerate it (`GATE_SNAPSHOT=update`, see gates.md) — then read that diff as a gate change. A comment saying "X is enforced by Y" is checked by running Y (E32: one such comment hid 95 functions the React Compiler skipped).
|
||||||
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` and burn down epic by epic. Fixed some? Regenerate with `--writebaseline`.
|
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` and burn down epic by epic. Fixed some? Regenerate with `--writebaseline`.
|
||||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||||
|
|
||||||
@@ -46,11 +49,13 @@ how it gets built. Concretely:
|
|||||||
behind each option and the weight of the decision. These decision points are where the product owner
|
behind each option and the weight of the decision. These decision points are where the product owner
|
||||||
engages with the build and learns from it, so make them understandable — a reasoned menu, not a bare one.
|
engages with the build and learns from it, so make them understandable — a reasoned menu, not a bare one.
|
||||||
- **Mockups for design/UX decisions.** For any visual or interaction choice, show mockup options to
|
- **Mockups for design/UX decisions.** For any visual or interaction choice, show mockup options to
|
||||||
choose from before building.
|
choose from before building. How the UI looks and behaves is decided in
|
||||||
|
[`docs/DESIGN.md`](docs/DESIGN.md) (its variants render in the dev-only `/design-lab`): build to it,
|
||||||
|
and change it only through its decision log.
|
||||||
- **Challenge unsound requests.** If a request is technically ill-advised and you can back it with real
|
- **Challenge unsound requests.** If a request is technically ill-advised and you can back it with real
|
||||||
arguments, push back or propose an alternative — don't comply blindly.
|
arguments, push back or propose an alternative — don't comply blindly.
|
||||||
- **Verify UI in the browser before calling it done.** tsc/unit/DOM tests miss visual and interaction
|
- **Verify UI in the browser before calling it done.** tsc/unit/DOM tests miss visual and interaction
|
||||||
bugs; do a real visual check of any UI change.
|
bugs; do a real visual check of any UI change, and run DESIGN.md's four-step Conformance check.
|
||||||
- **Be terse everywhere else.** Outside those decision points, lead with the answer or decision. During
|
- **Be terse everywhere else.** Outside those decision points, lead with the answer or decision. During
|
||||||
implementation and debugging especially: no restating the same point reworded, no filler.
|
implementation and debugging especially: no restating the same point reworded, no filler.
|
||||||
|
|
||||||
@@ -58,7 +63,7 @@ how it gets built. Concretely:
|
|||||||
|
|
||||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `bodylimit`, `progress`,
|
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
||||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||||
@@ -75,34 +80,39 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
|||||||
|
|
||||||
## Canonical patterns — when you have a choice, pick THIS
|
## Canonical patterns — when you have a choice, pick THIS
|
||||||
|
|
||||||
| Area | Use | Not |
|
| Area | Use | Not |
|
||||||
| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||||
| Request body | a Pydantic model | `payload: dict` |
|
| Request body | a Pydantic model | `payload: dict` |
|
||||||
| Request SIZE | nothing — `bodylimit` caps every route (SEC-20); need more? add a ceiling to `ROUTE_MAX_BODY_BYTES` | `capped_body` on a route that does not read raw bytes itself |
|
| Request SIZE | nothing — `bodylimit` caps every route (SEC-20); need more? add a ceiling to `ROUTE_MAX_BODY_BYTES` | `capped_body` on a route that does not read raw bytes itself |
|
||||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||||
| FE action button | `ui/Button` — pick a `SIZES` rung | a hand-rolled `<button className="px-3 py-2 …">` |
|
| FE button | `ui/Button` (a `SIZES` rung) · `IconButton` · `Pressable` (a clickable card/row) · `Chip` · `DragHandle` | a raw `<button>` or `FocusSafeButton` outside the control primitives (lint-enforced) |
|
||||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
| FE text field / checkbox / slider | `ui/Input` · `Textarea` · `InnerInput` in a `fieldClass` frame · `Checkbox`/`CheckRow` · `Slider` | a raw `<input>`/`<textarea>` anywhere but those primitives (lint-enforced) |
|
||||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
| FE glass surface | a role: `SURFACE.panel` / `card` / `menu` / `popover` / `tooltip` (`ui/surfaces.ts`) | a bare `glass`/`glass-card`/`glass-menu` beside a hand-picked radius (lint-enforced) |
|
||||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
||||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||||
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||||
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||||
| FE effect that must not re-subscribe | `useEffectEvent` (React 19.2+), else a latest-value ref | an excused dep list asserting the same thing in prose |
|
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||||
| FE hook suppression | an inline `eslint-disable-next-line` NAMING the invariant | `eslint-suppressions.json` (deliberately absent — E27 S2) |
|
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
||||||
|
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
||||||
|
| FE effect that must not re-subscribe | `useEffectEvent` (React 19.2+); a handler or timer: `useLatest` / `useStableCallback` | a render-phase `ref.current = x` — the React Compiler skips the component |
|
||||||
|
| FE hook suppression | an inline `eslint-disable-next-line` NAMING the invariant | `eslint-suppressions.json` (deliberately absent — E27 S2) |
|
||||||
|
| FE popover / scroll-fade ref | destructure it: `const { setReference } = pop.refs`, then `ref={setReference}` | `ref={pop.refs.setReference}` / `ref={fade.ref}` — the compiler then treats the whole object as a ref and skips |
|
||||||
|
| FE async handler with cleanup | `try { … } catch { … }`, then the cleanup after it (steps that `return` early go in their own function); a conditional, loop or `?.` the `try` needs goes before it or into a module-level helper | `try/finally`, or a `?:` / `??` / loop / `?.` inside `try` — the React Compiler skips the whole component (and `react-hooks/todo` misses most of these) |
|
||||||
|
|
||||||
## House rules that bite (learned from real bugs)
|
## House rules that bite (learned from real bugs)
|
||||||
|
|
||||||
|
|||||||
+7
-1
@@ -53,8 +53,11 @@ RUN adduser --disabled-password --gecos "" appuser \
|
|||||||
# Harmless on hosts with no /dev/dri (the driver just goes unused). Deno: the JavaScript runtime yt-dlp
|
# Harmless on hosts with no /dev/dri (the driver just goes unused). Deno: the JavaScript runtime yt-dlp
|
||||||
# uses to solve YouTube's "n" signature challenge for the web player clients (required alongside the PO
|
# uses to solve YouTube's "n" signature challenge for the web player clients (required alongside the PO
|
||||||
# token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
# token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
||||||
|
# `apt-get upgrade` (E32): the base image is a snapshot, and the Debian security fixes released since it
|
||||||
|
# was cut are only in the archive — the first Trivy scan found one (pcre2) fixed upstream but not in it.
|
||||||
ARG DENO_VERSION=v2.9.1
|
ARG DENO_VERSION=v2.9.1
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
|
&& apt-get upgrade -y --no-install-recommends \
|
||||||
&& apt-get install -y --no-install-recommends ffmpeg intel-media-va-driver unzip curl ca-certificates \
|
&& apt-get install -y --no-install-recommends ffmpeg intel-media-va-driver unzip curl ca-certificates \
|
||||||
&& curl -fsSL "https://github.com/denoland/deno/releases/download/${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o /tmp/deno.zip \
|
&& curl -fsSL "https://github.com/denoland/deno/releases/download/${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o /tmp/deno.zip \
|
||||||
&& unzip -o /tmp/deno.zip -d /usr/local/bin \
|
&& unzip -o /tmp/deno.zip -d /usr/local/bin \
|
||||||
@@ -78,9 +81,12 @@ RUN printf 'Types: deb\nURIs: http://deb.debian.org/debian\nSuites: bookworm\nCo
|
|||||||
# Install from the PINNED lock (requirements.lock, generated from requirements.txt) so the image is
|
# Install from the PINNED lock (requirements.lock, generated from requirements.txt) so the image is
|
||||||
# byte-reproducible and the exact versions are the ones pip-audit scanned in CI — not whatever floats
|
# byte-reproducible and the exact versions are the ones pip-audit scanned in CI — not whatever floats
|
||||||
# up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
# up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
||||||
|
# pip itself is removed once it has installed the lock (E32): nothing runs it at runtime, and it carries
|
||||||
|
# its own VENDORED copies of urllib3, msgpack and setuptools, which Trivy reports as the image's own —
|
||||||
|
# an attack surface and a CVE stream for a tool the running app never calls.
|
||||||
COPY backend/requirements.lock .
|
COPY backend/requirements.lock .
|
||||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||||
pip install --upgrade pip && pip install -r requirements.lock
|
pip install --upgrade pip && pip install -r requirements.lock && pip uninstall -y pip
|
||||||
|
|
||||||
COPY --chown=appuser backend/ .
|
COPY --chown=appuser backend/ .
|
||||||
# Read at runtime by config.py as the app_version (a committed single source of truth).
|
# Read at runtime by config.py as the app_version (a committed single source of truth).
|
||||||
|
|||||||
@@ -548,64 +548,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"./app/plex/stream.py": [
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 51,
|
|
||||||
"endColumn": 55,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 23,
|
|
||||||
"endColumn": 25,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 18,
|
|
||||||
"endColumn": 31,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 75,
|
|
||||||
"endColumn": 79,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 10,
|
|
||||||
"endColumn": 23,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 10,
|
|
||||||
"endColumn": 12,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 10,
|
|
||||||
"endColumn": 17,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"./app/plex/sync.py": [
|
"./app/plex/sync.py": [
|
||||||
{
|
{
|
||||||
"code": "reportArgumentType",
|
"code": "reportArgumentType",
|
||||||
@@ -638,22 +580,6 @@
|
|||||||
"endColumn": 89,
|
"endColumn": 89,
|
||||||
"lineCount": 1
|
"lineCount": 1
|
||||||
}
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 27,
|
|
||||||
"endColumn": 33,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 17,
|
|
||||||
"endColumn": 18,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"./app/plex/tmdb.py": [
|
"./app/plex/tmdb.py": [
|
||||||
@@ -1658,24 +1584,6 @@
|
|||||||
"endColumn": 36,
|
"endColumn": 36,
|
||||||
"lineCount": 1
|
"lineCount": 1
|
||||||
}
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 46,
|
|
||||||
"endColumn": 52,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"./app/youtube/rss.py": [
|
|
||||||
{
|
|
||||||
"code": "reportGeneralTypeIssues",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 10,
|
|
||||||
"endColumn": 15,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
"""add tagline_hu to plex_items + plex_shows — the HU overlay's tagline (E30 S1)
|
||||||
|
|
||||||
|
Revision ID: 0091_plex_tagline_hu
|
||||||
|
Revises: 0090_debug_session_draft
|
||||||
|
Create Date: 2026-09-29
|
||||||
|
|
||||||
|
The Hungarian overlay section lists a tagline beside the title and summary; the info page showed the
|
||||||
|
English one to a Hungarian reader. NULL until the next overlay sync fills it (serving falls back to EN).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0091_plex_tagline_hu"
|
||||||
|
down_revision: str | None = "0090_debug_session_draft"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("plex_items", sa.Column("tagline_hu", sa.Text(), nullable=True))
|
||||||
|
op.add_column("plex_shows", sa.Column("tagline_hu", sa.Text(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("plex_shows", "tagline_hu")
|
||||||
|
op.drop_column("plex_items", "tagline_hu")
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
"""add biography_hu to plex_people — TMDB's Hungarian biography (E30 S4)
|
||||||
|
|
||||||
|
Revision ID: 0092_plex_person_biography_hu
|
||||||
|
Revises: 0091_plex_tagline_hu
|
||||||
|
Create Date: 2026-09-29
|
||||||
|
|
||||||
|
The person page showed a Hungarian reader the English biography although TMDB has a Hungarian one for
|
||||||
|
most people. The person fetch now appends `translations` and keeps the Hungarian biography beside the
|
||||||
|
English one. Already-enriched people are un-stamped, so their next page open refetches them once (the
|
||||||
|
same lazy path as a first open) and fills it; until then the English biography stands in.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0092_plex_person_biography_hu"
|
||||||
|
down_revision: str | None = "0091_plex_tagline_hu"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("plex_people", sa.Column("biography_hu", sa.Text(), nullable=True))
|
||||||
|
op.execute(
|
||||||
|
"UPDATE plex_people SET tmdb_enriched_at = NULL "
|
||||||
|
"WHERE tmdb_person_id IS NOT NULL AND tmdb_enriched_at IS NOT NULL"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("plex_people", "biography_hu")
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
"""re-fetch linked people — "Known for" ranked by votes, without guest visits (E30 S6)
|
||||||
|
|
||||||
|
Revision ID: 0093_plex_person_known_for_votes
|
||||||
|
Revises: 0092_plex_person_biography_hu
|
||||||
|
Create Date: 2026-09-30
|
||||||
|
|
||||||
|
A person's "Known for" pool was their credits ranked by TMDB popularity, so talk-show visits as
|
||||||
|
themselves crowded out the films (Tom Cruise: 14 of 15 were talk shows, none in the library). The pool is
|
||||||
|
now ranked by vote count, without guest visits, with directed works, and deeper. Already-enriched people
|
||||||
|
are un-stamped so the people_enrich job fetches them again; their biography stays on the row meanwhile.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0093_plex_person_known_for_votes"
|
||||||
|
down_revision: str | None = "0092_plex_person_biography_hu"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute(
|
||||||
|
"UPDATE plex_people SET tmdb_enriched_at = NULL "
|
||||||
|
"WHERE tmdb_person_id IS NOT NULL AND tmdb_enriched_at IS NOT NULL"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
pass # the watermark only says "fetched"; a refetch under the old ranking is not worth undoing to
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
"""Hungarian genres on every Plex row, not only on overlaid ones (E30 review)
|
||||||
|
|
||||||
|
Revision ID: 0094_plex_genres_hu_everywhere
|
||||||
|
Revises: 0093_plex_person_known_for_votes
|
||||||
|
Create Date: 2026-09-30
|
||||||
|
|
||||||
|
The HU genre facets and filter read `genres_hu`, which only the overlay pass wrote — so a library with
|
||||||
|
no HU twin, or a guid-less row, offered English genres a Hungarian filter could never match. The sync
|
||||||
|
now writes `genres_hu` for every row (plex/sync.py `_apply_facet_fields`); this fills the rows it has
|
||||||
|
not reached yet, from their own English list through the same map.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
from app.plex.genres import hu_genres
|
||||||
|
|
||||||
|
revision: str = "0094_plex_genres_hu_everywhere"
|
||||||
|
down_revision: str | None = "0093_plex_person_known_for_votes"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
conn = op.get_bind()
|
||||||
|
for table in ("plex_items", "plex_shows"):
|
||||||
|
rows = conn.execute(
|
||||||
|
sa.text(
|
||||||
|
f"SELECT id, genres FROM {table} "
|
||||||
|
"WHERE genres_hu IS NULL AND jsonb_typeof(genres) = 'array'"
|
||||||
|
)
|
||||||
|
).fetchall()
|
||||||
|
updates = [
|
||||||
|
{"id": rid, "g": json.dumps(hu)} for rid, genres in rows if (hu := hu_genres(genres))
|
||||||
|
]
|
||||||
|
if updates:
|
||||||
|
conn.execute(
|
||||||
|
sa.text(f"UPDATE {table} SET genres_hu = CAST(:g AS jsonb) WHERE id = :id"), updates
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
pass # derived from `genres`; the sync writes the same values again
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""videos: the per-channel index covers what the channel manager and the feed sidebar read (E31 S4d)
|
||||||
|
|
||||||
|
The channel manager (`GET /api/channels`) sums each subscribed channel's videos: count, last upload, total
|
||||||
|
duration. The feed sidebar (`/feed/summary`, `/feed/years`) counts the reader's videos per channel and per
|
||||||
|
year under the default filters: available, not via explore, not a short, not live. Every one of them found
|
||||||
|
its rows through `ix_videos_channel_pub_desc` (channel_id, published_at DESC NULLS LAST, id) and then read
|
||||||
|
each video's heap page for the rest: ~270k rows at ~2.5 KB each, 681 MB on prod. That stays in cache only
|
||||||
|
while nothing else needs the memory. Measured on prod (2026-10-02): `/api/channels` 51–61 ms warm, but
|
||||||
|
2.25 s and 1.96 s on a first open in the browser and 6.4 s under EXPLAIN with 30k blocks read from disk;
|
||||||
|
`/feed/summary` and `/feed/years` ~140 ms warm, ~800 ms cold.
|
||||||
|
|
||||||
|
The same key with those columns INCLUDEd makes them Index Only Scans. On a prod-copy localdev, after a
|
||||||
|
VACUUM, the manager's sums read 0 heap pages and the summary reads the 25 MB index instead of the heap.
|
||||||
|
The visibility map is what lets them skip the heap; prod's autovacuum kept it complete (relallvisible ==
|
||||||
|
relpages). The key columns are unchanged, so every reader of the old index (the per-channel feed, the FK
|
||||||
|
cascade, the sync upsert, channel detail) rides this one. The old index is dropped.
|
||||||
|
tests/test_channel_list_index_only.py holds the queries and this column list together.
|
||||||
|
|
||||||
|
CONCURRENTLY (autocommit_block) builds it without blocking a live prod. IF (NOT) EXISTS keeps the drift
|
||||||
|
test's downgrade → re-upgrade round trip idempotent. A re-run after a build that failed part-way rebuilds
|
||||||
|
the INVALID leftover, and the old index is dropped only once the new one is valid: a re-run that kept the
|
||||||
|
leftover and dropped the old one left `videos` with no usable channel_id index (E31 S5 review).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
revision: str = "0095_videos_channel_cover_index"
|
||||||
|
down_revision: str | None = "0094_plex_genres_hu_everywhere"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def _valid(name: str) -> bool | None:
|
||||||
|
"""The index's `indisvalid`, or None when there is no index of that name."""
|
||||||
|
row = (
|
||||||
|
op.get_bind()
|
||||||
|
.execute(
|
||||||
|
text(
|
||||||
|
"SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid "
|
||||||
|
"WHERE c.relname = :name"
|
||||||
|
),
|
||||||
|
{"name": name},
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
return None if row is None else bool(row[0])
|
||||||
|
|
||||||
|
|
||||||
|
def _build(name: str, ddl: str) -> None:
|
||||||
|
"""CREATE INDEX CONCURRENTLY `name` (`ddl` is what follows the name), re-runnable. A build that
|
||||||
|
failed part-way leaves an INVALID index under the name, and `IF NOT EXISTS` would keep it: the
|
||||||
|
planner never uses it, and nothing says so (E31 S5 review). Such a leftover is dropped and built
|
||||||
|
again."""
|
||||||
|
if _valid(name) is False:
|
||||||
|
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||||
|
op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} {ddl}")
|
||||||
|
|
||||||
|
|
||||||
|
def _swap(build: str, ddl: str, drop: str) -> None:
|
||||||
|
_build(build, ddl)
|
||||||
|
if not _valid(build):
|
||||||
|
raise RuntimeError(f"{build} did not build valid; {drop} is kept")
|
||||||
|
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {drop}")
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.get_context().autocommit_block():
|
||||||
|
_swap(
|
||||||
|
"ix_videos_channel_pub_cover",
|
||||||
|
"ON videos (channel_id, published_at DESC NULLS LAST, id) "
|
||||||
|
"INCLUDE (duration_seconds, unavailable_since, via_explore, is_short, live_status)",
|
||||||
|
"ix_videos_channel_pub_desc",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.get_context().autocommit_block():
|
||||||
|
_swap(
|
||||||
|
"ix_videos_channel_pub_desc",
|
||||||
|
"ON videos (channel_id, published_at DESC NULLS LAST, id ASC)",
|
||||||
|
"ix_videos_channel_pub_cover",
|
||||||
|
)
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
"""videos: an index per feed sort, so a sorted page stops after its rows (E31 S4e)
|
||||||
|
|
||||||
|
The feed's views, duration and title sorts had no index. Every page (the first and each next cursor)
|
||||||
|
read the whole videos heap, filtered it and kept a top-N: on prod ~350 ms a page for views/duration,
|
||||||
|
550–780 ms for title (the ICU comparison on every row), 2.3 s for page 2 when the copies of one request
|
||||||
|
queued. With an index in the exact ORDER BY (`feedquery.order_by`: the key, NULLS LAST, then id) the
|
||||||
|
planner walks it and stops after the page. On a prod-copy localdev every plain sort fell from 180–310 ms
|
||||||
|
to ~1 ms, deep pages (20) to 3–5 ms, and narrow filters (a tag, a channel, a year, nothing found) stayed
|
||||||
|
at 0.3–25 ms.
|
||||||
|
|
||||||
|
One index per direction: a backward scan of `DESC NULLS LAST` gives `ASC NULLS FIRST`, not the
|
||||||
|
`ASC NULLS LAST` the asc sorts ask for. The title pair is per reader language (`app.collation.ICU`):
|
||||||
|
an index only serves an ORDER BY in its own collation.
|
||||||
|
|
||||||
|
Cost: ~130 MB (localdev: 10 + 10 + 8 + 8 + 4 × 23). Writes stay cheap: view_count and duration are
|
||||||
|
written by the enrich, which rewrites indexed columns already (the revalidation job only touches
|
||||||
|
last_checked_at), so no HOT update is lost. A search combined with one of these sorts can now walk the
|
||||||
|
index too: a term matching ~2 % of the videos took 130–200 ms instead of 7 (still under the plain sort it
|
||||||
|
replaces); rarer and broader terms stay fast. Fencing the index off under a search made broad terms
|
||||||
|
(`m`, `the`) take 2 s, so it is left to the planner.
|
||||||
|
|
||||||
|
The title indexes are the schema's first ones on an ICU collation: an ICU upgrade in the Postgres image
|
||||||
|
reorders them, and `app.collation.heal_drift` rebuilds them at startup when it sees one.
|
||||||
|
tests/test_feed_sort_indexes.py holds each sort and its index together.
|
||||||
|
|
||||||
|
CONCURRENTLY (autocommit_block) builds them without blocking a live prod; IF (NOT) EXISTS keeps the drift
|
||||||
|
test's downgrade → re-upgrade round trip idempotent.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
revision: str = "0096_videos_sort_indexes"
|
||||||
|
down_revision: str | None = "0095_videos_channel_cover_index"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
# name → the indexed ORDER BY (feedquery.sort_keys + order_by: key NULLS LAST, then id).
|
||||||
|
INDEXES = {
|
||||||
|
"ix_videos_views_desc": "view_count DESC NULLS LAST, id",
|
||||||
|
"ix_videos_views_asc": "view_count ASC NULLS LAST, id",
|
||||||
|
"ix_videos_duration_desc": "duration_seconds DESC NULLS LAST, id",
|
||||||
|
"ix_videos_duration_asc": "duration_seconds ASC NULLS LAST, id",
|
||||||
|
"ix_videos_title_en": '(lower(title) COLLATE "en-x-icu") ASC NULLS LAST, id',
|
||||||
|
"ix_videos_title_en_desc": '(lower(title) COLLATE "en-x-icu") DESC NULLS LAST, id',
|
||||||
|
"ix_videos_title_hu": '(lower(title) COLLATE "hu-x-icu") ASC NULLS LAST, id',
|
||||||
|
"ix_videos_title_hu_desc": '(lower(title) COLLATE "hu-x-icu") DESC NULLS LAST, id',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _valid(name: str) -> bool | None:
|
||||||
|
"""The index's `indisvalid`, or None when there is no index of that name."""
|
||||||
|
row = (
|
||||||
|
op.get_bind()
|
||||||
|
.execute(
|
||||||
|
text(
|
||||||
|
"SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid "
|
||||||
|
"WHERE c.relname = :name"
|
||||||
|
),
|
||||||
|
{"name": name},
|
||||||
|
)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
return None if row is None else bool(row[0])
|
||||||
|
|
||||||
|
|
||||||
|
def _build(name: str, ddl: str) -> None:
|
||||||
|
"""CREATE INDEX CONCURRENTLY `name` (`ddl` is what follows the name), re-runnable. A build that
|
||||||
|
failed part-way leaves an INVALID index under the name, and `IF NOT EXISTS` would keep it: the
|
||||||
|
planner never uses it, and nothing says so (E31 S5 review). Such a leftover is dropped and built
|
||||||
|
again."""
|
||||||
|
if _valid(name) is False:
|
||||||
|
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||||
|
op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} {ddl}")
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
with op.get_context().autocommit_block():
|
||||||
|
for name, columns in INDEXES.items():
|
||||||
|
_build(name, f"ON videos ({columns})")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
with op.get_context().autocommit_block():
|
||||||
|
for name in INDEXES:
|
||||||
|
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||||
@@ -23,7 +23,8 @@ Three layers, by *who* runs the code — not by feature:
|
|||||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
||||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||||
level) and `debugsession` (a recording that tags its user's lines, E16), … are single-file primitives many layers import. They depend on nothing above them (a core
|
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
||||||
|
reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||||
module must never import a `routes/…` module).
|
module must never import a `routes/…` module).
|
||||||
|
|
||||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||||
@@ -71,7 +72,9 @@ mutation it records, so an audit entry can never exist without its change (or vi
|
|||||||
## The gate
|
## The gate
|
||||||
|
|
||||||
Everything below must be green before a change is done (run via `siftlode check`):
|
Everything below must be green before a change is done (run via `siftlode check`):
|
||||||
`ruff` (backend) · `tsc` ×3 + `eslint` + `prettier` + `knip` (frontend) · `vitest` · `pytest`. The
|
`ruff` + `basedpyright` (backend) · the lock check + `tsc` ×3 + `eslint` + `prettier` + `knip` +
|
||||||
pytest lane runs in `backend/docker-compose.test.yml` — an ephemeral Postgres + the backend tree
|
`vite build` (frontend) · `vitest` · `pytest`. The pytest lane runs in `backend/docker-compose.test.yml`
|
||||||
bind-mounted — so the DB-backed integration tests (the `db` fixture) actually run; `SIFTLODE_TEST_
|
— an ephemeral Postgres + the backend tree bind-mounted — so the DB-backed integration tests (the `db`
|
||||||
REQUIRE_DB=1` there turns a would-be silent skip into a hard failure. See `docs/development.md`.
|
fixture) actually run; a missing database is a hard failure, not a silent skip, unless a hand run asks
|
||||||
|
for pure-logic only with `SIFTLODE_TEST_REQUIRE_DB=0`. See `docs/development.md`, and `docs/gates.md`
|
||||||
|
for what each gate deliberately leaves out.
|
||||||
|
|||||||
@@ -219,7 +219,7 @@ class BodyLimitMiddleware:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
await self.app(scope, counting_receive, guarded_send)
|
await self.app(scope, counting_receive, guarded_send)
|
||||||
except Exception: # noqa: BLE001 — only swallowed when WE caused it, see below
|
except Exception: # only swallowed when WE caused it, see below
|
||||||
# ClientDisconnect (or whatever a handler raises on a truncated read) is the expected
|
# ClientDisconnect (or whatever a handler raises on a truncated read) is the expected
|
||||||
# consequence of the disconnect we just faked, and the caller must get the 413 rather
|
# consequence of the disconnect we just faked, and the caller must get the 413 rather
|
||||||
# than a 500. Anything raised when we did NOT refuse is a real failure and is re-raised
|
# than a 500. Anything raised when we did NOT refuse is a real failure and is re-raised
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""Alphabetical order in the reader's language (E30 S3).
|
||||||
|
|
||||||
|
Why not the database default: the Postgres image is Alpine, and musl's `en_US.utf8` collation is BYTE
|
||||||
|
order — every accented initial sorted after "Z" ("Zorró < Életrajzi < Ötödik"), in every A→Z list the
|
||||||
|
server orders. Postgres ships the ICU collations on the same image (dev, test, prod alike), so a text
|
||||||
|
sort names one: Hungarian puts "Ö" after "O" and "Cs" after "Cu", English folds the accents in.
|
||||||
|
|
||||||
|
`collated` wraps an ORDER BY expression — and because a keyset cursor compares with the SAME expression
|
||||||
|
(`feedquery.sort_keys`), the cursor pages in the same order it sorted in. `sorted_words` orders a Python
|
||||||
|
list the same way through the database, so a list built in Python matches one built in SQL exactly.
|
||||||
|
|
||||||
|
The feed's title sorts ride indexes built in these collations (migration 0096), and an index keeps the
|
||||||
|
order of the ICU that built it. `postgres:16-alpine` is a floating tag: an image update can bring a newer
|
||||||
|
ICU, after which those indexes disagree with the comparisons a query makes (a keyset page skips or
|
||||||
|
repeats a title). `heal_drift` rebuilds them at startup once Postgres reports the version moved."""
|
||||||
|
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import Engine, SQLColumnExpression, column, func, select, text, values
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
from sqlalchemy.sql.elements import ColumnElement
|
||||||
|
from sqlalchemy.types import String
|
||||||
|
|
||||||
|
from app import logs
|
||||||
|
from app.mailtext import normalize
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
ICU: dict[str, str] = {"en": "en-x-icu", "hu": "hu-x-icu"}
|
||||||
|
|
||||||
|
|
||||||
|
def collation(lang: str | None) -> str:
|
||||||
|
"""The ICU collation for a UI language; anything unknown sorts as English."""
|
||||||
|
return ICU[normalize(lang)]
|
||||||
|
|
||||||
|
|
||||||
|
def collated(expr: SQLColumnExpression[Any], lang: str | None) -> ColumnElement[Any]:
|
||||||
|
"""`expr` compared in the reader's alphabet. Case-folded, so "apple" and "Apple" sit together."""
|
||||||
|
return func.lower(expr).collate(collation(lang))
|
||||||
|
|
||||||
|
|
||||||
|
def sorted_words(db: Session, words: list[str], lang: str | None) -> list[str]:
|
||||||
|
"""`words` in the reader's alphabetical order, by the same ICU collation as `collated`."""
|
||||||
|
if not words:
|
||||||
|
return []
|
||||||
|
w = values(column("w", String), name="words").data([(x,) for x in words])
|
||||||
|
return list(db.scalars(select(w.c.w).order_by(collated(w.c.w, lang), w.c.w)))
|
||||||
|
|
||||||
|
|
||||||
|
def heal_drift(engine: Engine) -> list[str]:
|
||||||
|
"""Rebuild every index on an ICU collation whose library version moved, then record the new version.
|
||||||
|
Returns the rebuilt indexes. CONCURRENTLY: the feed keeps reading while it runs."""
|
||||||
|
with engine.connect().execution_options(isolation_level="AUTOCOMMIT") as conn:
|
||||||
|
drifted = conn.execute(
|
||||||
|
text(
|
||||||
|
"SELECT oid, collname FROM pg_collation WHERE collname = ANY(:names) "
|
||||||
|
"AND collversion IS DISTINCT FROM pg_collation_actual_version(oid)"
|
||||||
|
),
|
||||||
|
{"names": list(ICU.values())},
|
||||||
|
).all()
|
||||||
|
if not drifted:
|
||||||
|
return []
|
||||||
|
indexes = list(
|
||||||
|
conn.execute(
|
||||||
|
text(
|
||||||
|
"SELECT DISTINCT d.objid::regclass::text FROM pg_depend d "
|
||||||
|
"JOIN pg_class c ON c.oid = d.objid AND c.relkind = 'i' "
|
||||||
|
"WHERE d.classid = 'pg_class'::regclass "
|
||||||
|
"AND d.refclassid = 'pg_collation'::regclass AND d.refobjid = ANY(:oids)"
|
||||||
|
),
|
||||||
|
{"oids": [row.oid for row in drifted]},
|
||||||
|
).scalars()
|
||||||
|
)
|
||||||
|
log.warning(
|
||||||
|
"ICU collation version changed (%s): rebuilding %d indexes",
|
||||||
|
", ".join(row.collname for row in drifted),
|
||||||
|
len(indexes),
|
||||||
|
)
|
||||||
|
for name in indexes:
|
||||||
|
conn.execute(text(f"REINDEX INDEX CONCURRENTLY {name}"))
|
||||||
|
for row in drifted:
|
||||||
|
conn.execute(text(f'ALTER COLLATION "{row.collname}" REFRESH VERSION'))
|
||||||
|
return indexes
|
||||||
@@ -366,6 +366,11 @@ class Settings(BaseSettings):
|
|||||||
tmdb_enrich_interval_min: int = 120
|
tmdb_enrich_interval_min: int = 120
|
||||||
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||||
tmdb_enrich_batch: int = 100
|
tmdb_enrich_batch: int = 100
|
||||||
|
# How often the people job (E30 S6) fetches a batch of person bios from TMDB and portraits through Plex,
|
||||||
|
# in minutes, and how many of each per run: ~18k linked people at 300 per 10 min is about ten hours
|
||||||
|
# once, then only the newcomers.
|
||||||
|
people_enrich_interval_min: int = 10
|
||||||
|
people_enrich_batch: int = 300
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def allowed_email_set(self) -> set[str]:
|
def allowed_email_set(self) -> set[str]:
|
||||||
|
|||||||
+3
-1
@@ -12,7 +12,9 @@ class Base(DeclarativeBase):
|
|||||||
|
|
||||||
# Pool sized above the sync-endpoint threadpool (~40) so a burst of short DB sessions can't starve on
|
# Pool sized above the sync-endpoint threadpool (~40) so a burst of short DB sessions can't starve on
|
||||||
# checkout. The image proxies deliberately DON'T hold a connection during their slow external fetch
|
# checkout. The image proxies deliberately DON'T hold a connection during their slow external fetch
|
||||||
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones.
|
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones. The one
|
||||||
|
# long hold — a playlist sync's advisory claim, for a whole push — opens its own unpooled connection
|
||||||
|
# (sync/playlists.py `_claim_engine`) instead of drawing a third one from here.
|
||||||
engine = create_engine(
|
engine = create_engine(
|
||||||
settings.database_url,
|
settings.database_url,
|
||||||
pool_pre_ping=True,
|
pool_pre_ping=True,
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ def pick(
|
|||||||
shelf: list[dict] | None = None,
|
shelf: list[dict] | None = None,
|
||||||
exclude: set[str] | None = None,
|
exclude: set[str] | None = None,
|
||||||
start: int = 0,
|
start: int = 0,
|
||||||
|
lang: str,
|
||||||
) -> dict | None:
|
) -> dict | None:
|
||||||
"""`{"tier": ..., "card": ...}`, or None when nothing at all fits.
|
"""`{"tier": ..., "card": ...}`, or None when nothing at all fits.
|
||||||
|
|
||||||
@@ -70,8 +71,8 @@ def pick(
|
|||||||
skip = exclude or set()
|
skip = exclude or set()
|
||||||
|
|
||||||
def _resume() -> dict | None:
|
def _resume() -> dict | None:
|
||||||
for card in rails.continue_watching(db, user) if resume is None else resume:
|
for card in rails.continue_watching(db, user, lang=lang) if resume is None else resume:
|
||||||
playable = rails.playable(db, user.id, card)
|
playable = rails.playable(db, user.id, card, lang=lang)
|
||||||
# Excluded on the card the USER saw (the show), not only on what it resolves to, so a
|
# Excluded on the card the USER saw (the show), not only on what it resolves to, so a
|
||||||
# rerolled-past show does not come straight back as its own next episode.
|
# rerolled-past show does not come straight back as its own next episode.
|
||||||
if cards.key(card) in skip or (playable is not None and cards.key(playable) in skip):
|
if cards.key(card) in skip or (playable is not None and cards.key(playable) in skip):
|
||||||
@@ -82,7 +83,7 @@ def pick(
|
|||||||
|
|
||||||
def _comfort() -> dict | None:
|
def _comfort() -> dict | None:
|
||||||
if shelf is None:
|
if shelf is None:
|
||||||
own = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
own = pins.shelf(db, user, include_plex=rails.plex_available(db), lang=lang)
|
||||||
else:
|
else:
|
||||||
own = shelf
|
own = shelf
|
||||||
for card in pins.playables(own):
|
for card in pins.playables(own):
|
||||||
@@ -93,7 +94,7 @@ def pick(
|
|||||||
def _surprise() -> dict | None:
|
def _surprise() -> dict | None:
|
||||||
# A wider draw than the one card we need: the shuffle is seeded, so without the headroom an
|
# A wider draw than the one card we need: the shuffle is seeded, so without the headroom an
|
||||||
# excluded pick would leave the tier empty rather than move past it.
|
# excluded pick would leave the tier empty rather than move past it.
|
||||||
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW):
|
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW, lang=lang):
|
||||||
if cards.key(card) not in skip:
|
if cards.key(card) not in skip:
|
||||||
return card
|
return card
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -170,7 +170,7 @@ class _Batch:
|
|||||||
return b
|
return b
|
||||||
|
|
||||||
|
|
||||||
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict | None:
|
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch, lang: str) -> dict | None:
|
||||||
"""One pin as a unified card with its resolved `next`. None when the pinned row has gone (the FK
|
"""One pin as a unified card with its resolved `next`. None when the pinned row has gone (the FK
|
||||||
cascade removes the pin itself, so this is only the window between a delete and a refetch)."""
|
cascade removes the pin itself, so this is only the window between a delete and a refetch)."""
|
||||||
card: dict | None = None
|
card: dict | None = None
|
||||||
@@ -196,7 +196,7 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict
|
|||||||
if it is None:
|
if it is None:
|
||||||
return None
|
return None
|
||||||
st = rollup.item_state_map(db, user.id, [it.id]).get(it.id)
|
st = rollup.item_state_map(db, user.id, [it.id]).get(it.id)
|
||||||
card = cards.from_plex(leaf_card(it, st))
|
card = cards.from_plex(leaf_card(it, st, lang=lang))
|
||||||
nxt = dict(card) # a movie or a single episode plays itself
|
nxt = dict(card) # a movie or a single episode plays itself
|
||||||
|
|
||||||
elif pin.plex_show_id is not None:
|
elif pin.plex_show_id is not None:
|
||||||
@@ -204,8 +204,8 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict
|
|||||||
if sh is None:
|
if sh is None:
|
||||||
return None
|
return None
|
||||||
status = rollup.state_map(db, user.id, [sh.id]).get(sh.id, "new")
|
status = rollup.state_map(db, user.id, [sh.id]).get(sh.id, "new")
|
||||||
card = cards.from_plex(show_card(sh, status))
|
card = cards.from_plex(show_card(sh, status, lang=lang))
|
||||||
ep = queries.next_episode(db, user.id, sh.id)
|
ep = queries.next_episode(db, user.id, sh.id, lang=lang)
|
||||||
nxt = cards.from_plex(ep) if ep else None
|
nxt = cards.from_plex(ep) if ep else None
|
||||||
|
|
||||||
elif pin.playlist_id is not None:
|
elif pin.playlist_id is not None:
|
||||||
@@ -229,7 +229,7 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict
|
|||||||
_PLEX_COLUMNS = ("plex_item_id", "plex_show_id")
|
_PLEX_COLUMNS = ("plex_item_id", "plex_show_id")
|
||||||
|
|
||||||
|
|
||||||
def shelf(db: Session, user: User, *, include_plex: bool = True) -> list[dict]:
|
def shelf(db: Session, user: User, *, include_plex: bool = True, lang: str) -> list[dict]:
|
||||||
"""The whole shelf, in the user's own order.
|
"""The whole shelf, in the user's own order.
|
||||||
|
|
||||||
`include_plex=False` is the Home surface with the deployment's Plex toggle off (the owner's call,
|
`include_plex=False` is the Home surface with the deployment's Plex toggle off (the owner's call,
|
||||||
@@ -243,7 +243,7 @@ def shelf(db: Session, user: User, *, include_plex: bool = True) -> list[dict]:
|
|||||||
pins = [p for p in pins if all(getattr(p, c) is None for c in _PLEX_COLUMNS)]
|
pins = [p for p in pins if all(getattr(p, c) is None for c in _PLEX_COLUMNS)]
|
||||||
batch = _Batch.load(db, user, pins)
|
batch = _Batch.load(db, user, pins)
|
||||||
for pin in pins:
|
for pin in pins:
|
||||||
card = _shelf_card(db, user, pin, batch)
|
card = _shelf_card(db, user, pin, batch, lang)
|
||||||
if card is not None:
|
if card is not None:
|
||||||
out.append(card)
|
out.append(card)
|
||||||
return out
|
return out
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ from app.models import (
|
|||||||
VideoState,
|
VideoState,
|
||||||
)
|
)
|
||||||
from app.plex import rollup
|
from app.plex import rollup
|
||||||
from app.plex.cards import leaf_card, movie_card, show_card
|
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||||
from app.plex.query import apply_show_state_filter, content_libs
|
from app.plex.query import apply_show_state_filter, content_libs
|
||||||
|
|
||||||
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
|
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
|
||||||
@@ -221,7 +221,9 @@ def _unseen_movie_conds(uid: int):
|
|||||||
return ~seen
|
return ~seen
|
||||||
|
|
||||||
|
|
||||||
def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tuple[object, dict]]:
|
def plex_in_progress(
|
||||||
|
db: Session, uid: int, limit: int = RAIL_LIMIT, *, lang: str
|
||||||
|
) -> list[tuple[object, dict]]:
|
||||||
"""In-progress movies + partially-watched shows as (progressed_at, card).
|
"""In-progress movies + partially-watched shows as (progressed_at, card).
|
||||||
|
|
||||||
Mirrors `routes/plex/discover.py::_continue_watching` — same `PlexState` predicate for movies (a
|
Mirrors `routes/plex/discover.py::_continue_watching` — same `PlexState` predicate for movies (a
|
||||||
@@ -249,7 +251,7 @@ def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tup
|
|||||||
.limit(limit)
|
.limit(limit)
|
||||||
)
|
)
|
||||||
for it, st in db.execute(mq).all():
|
for it, st in db.execute(mq).all():
|
||||||
out.append((st.progress_updated_at, movie_card(it, st)))
|
out.append((st.progress_updated_at, movie_card(it, st, lang=lang)))
|
||||||
|
|
||||||
if show_libs:
|
if show_libs:
|
||||||
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
||||||
@@ -263,12 +265,14 @@ def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tup
|
|||||||
for sh, resumed_at in db.execute(
|
for sh, resumed_at in db.execute(
|
||||||
sq.order_by(prog.c.resumed_at.desc().nullslast()).limit(limit)
|
sq.order_by(prog.c.resumed_at.desc().nullslast()).limit(limit)
|
||||||
).all():
|
).all():
|
||||||
out.append((resumed_at, show_card(sh, "in_progress")))
|
out.append((resumed_at, show_card(sh, "in_progress", lang=lang)))
|
||||||
|
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) -> list[dict]:
|
def plex_surprise(
|
||||||
|
db: Session, uid: int, budget: str, seed: int, limit: int, *, lang: str
|
||||||
|
) -> list[dict]:
|
||||||
"""Unseen playable movies inside the budget, in seeded-shuffle order.
|
"""Unseen playable movies inside the budget, in seeded-shuffle order.
|
||||||
|
|
||||||
Movies only. A show is not a thing you can start playing, and resolving every candidate show to
|
Movies only. A show is not a thing you can start playing, and resolving every candidate show to
|
||||||
@@ -300,14 +304,14 @@ def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) ->
|
|||||||
q = q.where(PlexItem.duration_s <= hi)
|
q = q.where(PlexItem.duration_s <= hi)
|
||||||
items = db.execute(q).scalars().all()
|
items = db.execute(q).scalars().all()
|
||||||
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||||
return [movie_card(it, states.get(it.id)) for it in items]
|
return [movie_card(it, states.get(it.id), lang=lang) for it in items]
|
||||||
|
|
||||||
|
|
||||||
# Watch states that take an episode out of the running for "what would play next".
|
# Watch states that take an episode out of the running for "what would play next".
|
||||||
_DONE = ("watched", "hidden")
|
_DONE = ("watched", "hidden")
|
||||||
|
|
||||||
|
|
||||||
def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
def next_episode(db: Session, uid: int, show_id: int, *, lang: str) -> dict | None:
|
||||||
"""What "put that show on" means: the episode in progress, else the first unwatched one, else the
|
"""What "put that show on" means: the episode in progress, else the first unwatched one, else the
|
||||||
first — over the same `(season, episode)` broadcast ordering `detail.py::_episode_nav` uses.
|
first — over the same `(season, episode)` broadcast ordering `detail.py::_episode_nav` uses.
|
||||||
|
|
||||||
@@ -357,5 +361,9 @@ def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
|||||||
ep, st = pick
|
ep, st = pick
|
||||||
show = db.get(PlexShow, show_id)
|
show = db.get(PlexShow, show_id)
|
||||||
return leaf_card(
|
return leaf_card(
|
||||||
ep, st, show_title=show.title if show else None, show_rk=show.rating_key if show else None
|
ep,
|
||||||
|
st,
|
||||||
|
show_title=display_title(show, lang) if show else None,
|
||||||
|
show_rk=show.rating_key if show else None,
|
||||||
|
lang=lang,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ def plex_available(db: Session) -> bool:
|
|||||||
return sysconfig.get_bool(db, "plex_enabled")
|
return sysconfig.get_bool(db, "plex_enabled")
|
||||||
|
|
||||||
|
|
||||||
def playable(db: Session, uid: int, card: dict) -> dict | None:
|
def playable(db: Session, uid: int, card: dict, *, lang: str) -> dict | None:
|
||||||
"""The thing that would actually START from this card — itself for anything with a runtime, and
|
"""The thing that would actually START from this card — itself for anything with a runtime, and
|
||||||
for a Plex SHOW the episode that is next (in progress, else the first unwatched, else the first).
|
for a Plex SHOW the episode that is next (in progress, else the first unwatched, else the first).
|
||||||
|
|
||||||
@@ -40,11 +40,11 @@ def playable(db: Session, uid: int, card: dict) -> dict | None:
|
|||||||
sh = by_rating_key(db, PlexShow, card["id"])
|
sh = by_rating_key(db, PlexShow, card["id"])
|
||||||
if sh is None:
|
if sh is None:
|
||||||
return None
|
return None
|
||||||
ep = queries.next_episode(db, uid, sh.id)
|
ep = queries.next_episode(db, uid, sh.id, lang=lang)
|
||||||
return cards.from_plex(ep) if ep else None
|
return cards.from_plex(ep) if ep else None
|
||||||
|
|
||||||
|
|
||||||
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[dict]:
|
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT, *, lang: str) -> list[dict]:
|
||||||
"""One resume rail across both sources, most recently progressed first.
|
"""One resume rail across both sources, most recently progressed first.
|
||||||
|
|
||||||
A Plex SHOW enters as a show CARD — that is what the Plex rail shows, and it is the artwork the
|
A Plex SHOW enters as a show CARD — that is what the Plex rail shows, and it is the artwork the
|
||||||
@@ -57,17 +57,22 @@ def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[
|
|||||||
(ts, cards.from_video(r)) for ts, r in queries.yt_in_progress(db, user, limit)
|
(ts, cards.from_video(r)) for ts, r in queries.yt_in_progress(db, user, limit)
|
||||||
]
|
]
|
||||||
if plex_available(db):
|
if plex_available(db):
|
||||||
rows += [(ts, cards.from_plex(c)) for ts, c in queries.plex_in_progress(db, user.id, limit)]
|
rows += [
|
||||||
|
(ts, cards.from_plex(c))
|
||||||
|
for ts, c in queries.plex_in_progress(db, user.id, limit, lang=lang)
|
||||||
|
]
|
||||||
rows.sort(key=lambda r: r[0] or NO_TS, reverse=True)
|
rows.sort(key=lambda r: r[0] or NO_TS, reverse=True)
|
||||||
out = [card for _, card in rows[:limit]]
|
out = [card for _, card in rows[:limit]]
|
||||||
# Resolve AFTER the slice: only the rows that survive it are worth a query.
|
# Resolve AFTER the slice: only the rows that survive it are worth a query.
|
||||||
for card in out:
|
for card in out:
|
||||||
if card["kind"] == "show":
|
if card["kind"] == "show":
|
||||||
card["next"] = playable(db, user.id, card)
|
card["next"] = playable(db, user.id, card, lang=lang)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) -> list[dict]:
|
def surprise(
|
||||||
|
db: Session, user: User, budget: str, seed: int, limit: int = 3, *, lang: str
|
||||||
|
) -> list[dict]:
|
||||||
"""The hero's draw pool: unseen things that fit the budget, from both sources under ONE seed.
|
"""The hero's draw pool: unseen things that fit the budget, from both sources under ONE seed.
|
||||||
|
|
||||||
Interleaved rather than concatenated — the same alternation `routes/plex/discover.py` uses for its
|
Interleaved rather than concatenated — the same alternation `routes/plex/discover.py` uses for its
|
||||||
@@ -83,7 +88,9 @@ def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) ->
|
|||||||
(cards.from_video, queries.yt_surprise(db, user, budget, seed, limit)),
|
(cards.from_video, queries.yt_surprise(db, user, budget, seed, limit)),
|
||||||
(
|
(
|
||||||
cards.from_plex,
|
cards.from_plex,
|
||||||
queries.plex_surprise(db, user.id, budget, seed, limit) if plex_available(db) else [],
|
queries.plex_surprise(db, user.id, budget, seed, limit, lang=lang)
|
||||||
|
if plex_available(db)
|
||||||
|
else [],
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
if seed % 2:
|
if seed % 2:
|
||||||
|
|||||||
@@ -400,7 +400,7 @@ def escalate_kill(proc: subprocess.Popen, terminate, kill, grace_s: float, label
|
|||||||
try:
|
try:
|
||||||
proc.wait(timeout=grace_s)
|
proc.wait(timeout=grace_s)
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
# noqa TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
# TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
||||||
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
||||||
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
||||||
|
|
||||||
@@ -456,7 +456,7 @@ def run_ffmpeg(
|
|||||||
try:
|
try:
|
||||||
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
|
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
|
||||||
except subprocess.TimeoutExpired: # pipes closed but the process lingers
|
except subprocess.TimeoutExpired: # pipes closed but the process lingers
|
||||||
raise StatedFailure("ffmpeg did not exit after closing its output")
|
raise StatedFailure("ffmpeg did not exit after closing its output") from None
|
||||||
except BaseException:
|
except BaseException:
|
||||||
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
|
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
|
||||||
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
|
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ def main() -> None:
|
|||||||
logs.configure_logging("runner")
|
logs.configure_logging("runner")
|
||||||
_deprioritize()
|
_deprioritize()
|
||||||
if len(sys.argv) != 2:
|
if len(sys.argv) != 2:
|
||||||
# noqa T201: a usage line is for the person who typed the command. Logging is already
|
# T201: a usage line is for the person who typed the command. Logging is already
|
||||||
# configured here, and in a deployment it writes JSON records — the one reader of this
|
# configured here, and in a deployment it writes JSON records — the one reader of this
|
||||||
# line wants the bare sentence, not a record with a request id they do not have.
|
# line wants the bare sentence, not a record with a request id they do not have.
|
||||||
print( # noqa: T201
|
print( # noqa: T201
|
||||||
@@ -104,7 +104,7 @@ def _run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id) ->
|
|||||||
else:
|
else:
|
||||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||||
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
||||||
# noqa TRY400: the stack goes with it scrubbed (`scrubbed_stack`); `exc_info` is raw.
|
# TRY400: the stack goes with it scrubbed (`scrubbed_stack`); `exc_info` is raw.
|
||||||
log.error( # noqa: TRY400
|
log.error( # noqa: TRY400
|
||||||
"the download child for job %s failed",
|
"the download child for job %s failed",
|
||||||
job_id,
|
job_id,
|
||||||
|
|||||||
+106
-22
@@ -12,9 +12,10 @@ import binascii
|
|||||||
import json
|
import json
|
||||||
from datetime import date, datetime, timedelta, timezone
|
from datetime import date, datetime, timedelta, timezone
|
||||||
|
|
||||||
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
|
from sqlalchemy import BigInteger, Select, and_, case, cast, false, func, or_, select
|
||||||
from sqlalchemy.orm import Session, aliased
|
from sqlalchemy.orm import Session, aliased
|
||||||
|
|
||||||
|
from app.collation import collated
|
||||||
from app.models import (
|
from app.models import (
|
||||||
LIVE_OR_UPCOMING,
|
LIVE_OR_UPCOMING,
|
||||||
BlockedChannel,
|
BlockedChannel,
|
||||||
@@ -105,15 +106,30 @@ def _tag_visible(user: User):
|
|||||||
return or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
return or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||||
|
|
||||||
|
|
||||||
def tags_by_category(db: Session, tag_ids: list[int]) -> dict[str, list[int]]:
|
# The pseudo-category of a selected tag id no Tag row answers (a saved view or the feed state outlives
|
||||||
|
# a deleted tag). It filters like a category whose tags no channel carries — to nothing, in the SQL and
|
||||||
|
# in feed_summary's mirror alike — instead of the id being dropped, which showed the WHOLE feed under
|
||||||
|
# the view's old name (E32 S2). No real category is called this (language/topic/other).
|
||||||
|
MISSING_TAG_CATEGORY = "missing"
|
||||||
|
|
||||||
|
|
||||||
|
def tags_by_category(db: Session, user: User, tag_ids: list[int]) -> dict[str, list[int]]:
|
||||||
"""Group tag ids by their category (language/topic/other). The single source for the category split
|
"""Group tag ids by their category (language/topic/other). The single source for the category split
|
||||||
shared by the feed's tag WHERE (filtered_query) and the facet/count summary (feed_summary), so the
|
shared by the feed's tag WHERE (filtered_query) and the facet/count summary (feed_summary), so the
|
||||||
two can't disagree on which selected tag belongs to which category."""
|
two can't disagree on which selected tag belongs to which category. An unknown id lands in
|
||||||
|
MISSING_TAG_CATEGORY — and so does another user's private tag: resolving it to its real category
|
||||||
|
told the caller which foreign ids exist (E32 review)."""
|
||||||
if not tag_ids:
|
if not tag_ids:
|
||||||
return {} # skip the pointless `WHERE id IN ()` round-trip on the no-tag-filter hot path
|
return {} # skip the pointless `WHERE id IN ()` round-trip on the no-tag-filter hot path
|
||||||
out: dict[str, list[int]] = {}
|
out: dict[str, list[int]] = {}
|
||||||
for tid, cat in db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids))).all():
|
visible = or_(Tag.user_id.is_(None), Tag.user_id == user.id)
|
||||||
|
for tid, cat in db.execute(
|
||||||
|
select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids), visible)
|
||||||
|
).all():
|
||||||
out.setdefault(cat, []).append(tid)
|
out.setdefault(cat, []).append(tid)
|
||||||
|
known = {tid for ids in out.values() for tid in ids}
|
||||||
|
if missing := [tid for tid in tag_ids if tid not in known]:
|
||||||
|
out[MISSING_TAG_CATEGORY] = missing
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
@@ -264,7 +280,8 @@ def filtered_query(
|
|||||||
# the queries that surfaced the video > description; see Video.search_vector). YouTube-like
|
# the queries that surfaced the video > description; see Video.search_vector). YouTube-like
|
||||||
# "fuzzy" matching: word-order-independent, multi-word AND, prefix on the word being typed,
|
# "fuzzy" matching: word-order-independent, multi-word AND, prefix on the word being typed,
|
||||||
# accent-insensitive (unaccent_simple). The channel name still matches as a substring so
|
# accent-insensitive (unaccent_simple). The channel name still matches as a substring so
|
||||||
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort.
|
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort, over the
|
||||||
|
# newest RANK_WINDOW matches only (see there).
|
||||||
rank_expr = None
|
rank_expr = None
|
||||||
if q:
|
if q:
|
||||||
# Resolve the channels whose NAME matches up front (there are only a few hundred channels, so
|
# Resolve the channels whose NAME matches up front (there are only a few hundred channels, so
|
||||||
@@ -302,7 +319,7 @@ def filtered_query(
|
|||||||
# ALL its selected tags. ⚠️ feed_summary.passes() reimplements this exact rule in Python for the
|
# ALL its selected tags. ⚠️ feed_summary.passes() reimplements this exact rule in Python for the
|
||||||
# sidebar count/facets — keep the two in lockstep (equivalence-tested by
|
# sidebar count/facets — keep the two in lockstep (equivalence-tested by
|
||||||
# test_feed_summary_matches_count_plus_facets).
|
# test_feed_summary_matches_count_plus_facets).
|
||||||
by_category = tags_by_category(db, tags)
|
by_category = tags_by_category(db, user, tags)
|
||||||
# Facet counting drops the category being counted so its own chips don't zero each
|
# Facet counting drops the category being counted so its own chips don't zero each
|
||||||
# other out (standard drill-down faceting: a category's count ignores its own
|
# other out (standard drill-down faceting: a category's count ignores its own
|
||||||
# selections but still honours the other categories' filters).
|
# selections but still honours the other categories' filters).
|
||||||
@@ -393,7 +410,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
|||||||
return {"count": 0, "facets": {}}
|
return {"count": 0, "facets": {}}
|
||||||
|
|
||||||
# channel -> its visible tag ids, plus each tag's category (for grouping the facet output).
|
# channel -> its visible tag ids, plus each tag's category (for grouping the facet output).
|
||||||
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
visible = _tag_visible(user)
|
||||||
channel_tags: dict[str, set[int]] = {}
|
channel_tags: dict[str, set[int]] = {}
|
||||||
tag_category: dict[int, str] = {}
|
tag_category: dict[int, str] = {}
|
||||||
for cid, tid, cat in db.execute(
|
for cid, tid, cat in db.execute(
|
||||||
@@ -408,7 +425,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
|||||||
# base channel carries still filters everything out for its category — matching the SQL `IN (empty)`
|
# base channel carries still filters everything out for its category — matching the SQL `IN (empty)`
|
||||||
# behaviour).
|
# behaviour).
|
||||||
selected_by_cat: dict[str, set[int]] = {
|
selected_by_cat: dict[str, set[int]] = {
|
||||||
cat: set(ids) for cat, ids in tags_by_category(db, params.get("tags") or []).items()
|
cat: set(ids) for cat, ids in tags_by_category(db, user, params.get("tags") or []).items()
|
||||||
}
|
}
|
||||||
|
|
||||||
def passes(ch_tags: set[int], exclude_cat: str | None) -> bool:
|
def passes(ch_tags: set[int], exclude_cat: str | None) -> bool:
|
||||||
@@ -436,7 +453,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
|||||||
# Drop this category's own selections so its chips keep independent counts; a conjunctive
|
# Drop this category's own selections so its chips keep independent counts; a conjunctive
|
||||||
# topic-AND keeps them (each remaining chip narrows to channels that ALSO have the selected).
|
# topic-AND keeps them (each remaining chip narrows to channels that ALSO have the selected).
|
||||||
exclude = None if tag_modes.get(category, "or") == "and" else category
|
exclude = None if tag_modes.get(category, "or") == "and" else category
|
||||||
for cid, ch_tags in channel_tags.items():
|
for ch_tags in channel_tags.values():
|
||||||
if not passes(ch_tags, exclude):
|
if not passes(ch_tags, exclude):
|
||||||
continue
|
continue
|
||||||
for tid in ch_tags:
|
for tid in ch_tags:
|
||||||
@@ -478,8 +495,10 @@ def feed_years(db: Session, user: User, **params) -> list[dict]:
|
|||||||
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
|
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
|
||||||
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
||||||
# (OFFSET would shift and duplicate/skip rows).
|
# (OFFSET would shift and duplicate/skip rows).
|
||||||
def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) -> list[dict]:
|
||||||
|
"""`lang` orders the title sorts in the reader's alphabet (`collation`); the other keys ignore it."""
|
||||||
pub = {"expr": Video.published_at, "asc": False, "nullable": False, "kind": "dt"}
|
pub = {"expr": Video.published_at, "asc": False, "nullable": False, "kind": "dt"}
|
||||||
|
title = collated(Video.title, lang)
|
||||||
table: dict[str, list[dict]] = {
|
table: dict[str, list[dict]] = {
|
||||||
"newest": [pub],
|
"newest": [pub],
|
||||||
"oldest": [{**pub, "asc": True}],
|
"oldest": [{**pub, "asc": True}],
|
||||||
@@ -491,10 +510,8 @@ def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
|||||||
"duration_asc": [
|
"duration_asc": [
|
||||||
{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}
|
{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}
|
||||||
],
|
],
|
||||||
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
|
"title": [{"expr": title, "asc": True, "nullable": True, "kind": "str"}],
|
||||||
"title_desc": [
|
"title_desc": [{"expr": title, "asc": False, "nullable": True, "kind": "str"}],
|
||||||
{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}
|
|
||||||
],
|
|
||||||
"subscribers": [
|
"subscribers": [
|
||||||
{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}
|
{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}
|
||||||
],
|
],
|
||||||
@@ -542,6 +559,54 @@ def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
|||||||
return table.get(sort) or table["newest"]
|
return table.get(sort) or table["newest"]
|
||||||
|
|
||||||
|
|
||||||
|
# Relevance ranks only the newest RANK_WINDOW matches (E31): ts_rank reads every match's tsvector, and
|
||||||
|
# a broad word matches most of the library ("a" ranked 193k rows in 6.8 s on prod, against 1.1 s to just
|
||||||
|
# find them). The rows past the window keep rank 0, so they follow the ranked ones newest first; a word
|
||||||
|
# that broad has no meaningful relevance order anyway, and a specific one fits the window whole.
|
||||||
|
RANK_WINDOW = 2000
|
||||||
|
|
||||||
|
|
||||||
|
def rank_window_start(db: Session, query: Select) -> datetime | None:
|
||||||
|
"""The published_at the ranked window starts at: the RANK_WINDOW-th newest match, None when fewer
|
||||||
|
match (rank them all). The newest-first walk stops after RANK_WINDOW matches (40 ms for "a")."""
|
||||||
|
return db.scalar(
|
||||||
|
query.with_only_columns(Video.published_at)
|
||||||
|
.order_by(Video.published_at.desc())
|
||||||
|
.offset(RANK_WINDOW - 1)
|
||||||
|
.limit(1)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def windowed_rank(rank_expr, since: datetime | None):
|
||||||
|
"""`rank_expr` inside the window, 0 past it. CASE evaluates ts_rank only for the rows inside."""
|
||||||
|
if since is None:
|
||||||
|
return rank_expr
|
||||||
|
return case((Video.published_at >= since, rank_expr), else_=0)
|
||||||
|
|
||||||
|
|
||||||
|
def windowed_relevance_rows(
|
||||||
|
db: Session, query: Select, keys: list[dict], window: datetime, cursor_id: str | None, n: int
|
||||||
|
) -> list:
|
||||||
|
"""Up to `n` rows of the windowed relevance order, read as its two halves: the window ranked, then
|
||||||
|
the rest newest first (every window row sorts before every row past it). As one ORDER BY the
|
||||||
|
planner computes the CASE for every match and sorts them all (2.7 s for "a" on prod); split, each
|
||||||
|
half rides an index (80 ms + 5 ms). With a cursor, `keys` carry its values (keyset_predicate)."""
|
||||||
|
pub_key = keys[1] # relevance: [rank, published_at]
|
||||||
|
rows: list = []
|
||||||
|
if cursor_id is None or pub_key["value"] >= window:
|
||||||
|
inside = query.where(Video.published_at >= window).order_by(*order_by(keys))
|
||||||
|
if cursor_id is not None:
|
||||||
|
inside = inside.where(keyset_predicate(keys, cursor_id))
|
||||||
|
rows = list(db.execute(inside.limit(n)).all())
|
||||||
|
if len(rows) < n:
|
||||||
|
newest = [pub_key] # past the window every rank is 0: the order is newest first
|
||||||
|
rest = query.where(Video.published_at < window).order_by(*order_by(newest))
|
||||||
|
if cursor_id is not None:
|
||||||
|
rest = rest.where(keyset_predicate(newest, cursor_id))
|
||||||
|
rows += db.execute(rest.limit(n - len(rows))).all()
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
def order_by(keys: list[dict]):
|
def order_by(keys: list[dict]):
|
||||||
cols = []
|
cols = []
|
||||||
for k in keys:
|
for k in keys:
|
||||||
@@ -588,26 +653,45 @@ def keyset_predicate(keys: list[dict], cursor_id: str):
|
|||||||
return rec(0)
|
return rec(0)
|
||||||
|
|
||||||
|
|
||||||
def encode_cursor(keys: list[dict], row) -> str:
|
def encode_cursor(keys: list[dict], row, window: datetime | None = None) -> str:
|
||||||
|
"""`window` is the relevance window's start: the cursor carries it, so every page ranks against the
|
||||||
|
window page one did. Recomputed per page, a video ingested mid-scroll would push the window's oldest
|
||||||
|
row to rank 0, behind the cursor, and the reader would see it twice."""
|
||||||
vals = []
|
vals = []
|
||||||
for i, k in enumerate(keys):
|
for i, k in enumerate(keys):
|
||||||
v = getattr(row, f"_k{i}")
|
v = getattr(row, f"_k{i}")
|
||||||
vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v)
|
vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v)
|
||||||
payload = json.dumps({"k": vals, "id": row.id})
|
body: dict = {"k": vals, "id": row.id}
|
||||||
|
if window is not None:
|
||||||
|
body["w"] = window.isoformat()
|
||||||
|
payload = json.dumps(body)
|
||||||
return base64.urlsafe_b64encode(payload.encode()).decode()
|
return base64.urlsafe_b64encode(payload.encode()).decode()
|
||||||
|
|
||||||
|
|
||||||
def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]:
|
def _cursor_dt(v):
|
||||||
|
"""A cursor's timestamp: aware or refused. The server writes only aware ones (timestamptz), and
|
||||||
|
windowed_relevance_rows compares them in Python, where a naive one beside an aware one raises — a
|
||||||
|
500 for a hand-made cursor instead of the 400 (E31 review 2)."""
|
||||||
|
if v is None:
|
||||||
|
return None
|
||||||
|
dt = datetime.fromisoformat(v)
|
||||||
|
if dt.tzinfo is None:
|
||||||
|
raise ValueError("naive cursor timestamp")
|
||||||
|
return dt
|
||||||
|
|
||||||
|
|
||||||
|
def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str, datetime | None]:
|
||||||
try:
|
try:
|
||||||
payload = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
payload = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||||
raw = payload["k"]
|
raw = payload["k"]
|
||||||
if len(raw) != len(keys):
|
if len(raw) != len(keys):
|
||||||
raise ValueError("key arity mismatch")
|
raise ValueError("key arity mismatch")
|
||||||
vals = [
|
vals = [_cursor_dt(v) if k["kind"] == "dt" else v for k, v in zip(keys, raw, strict=True)]
|
||||||
datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v
|
for k, v in zip(keys, vals, strict=True):
|
||||||
for k, v in zip(keys, raw)
|
if v is None and not k["nullable"]:
|
||||||
]
|
raise ValueError("null value for a non-nullable key")
|
||||||
return vals, payload["id"]
|
window = payload.get("w")
|
||||||
|
return vals, payload["id"], _cursor_dt(window) if window else None
|
||||||
except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc:
|
except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc:
|
||||||
# Framework-free: the route handler translates this into an HTTP 400 so the
|
# Framework-free: the route handler translates this into an HTTP 400 so the
|
||||||
# read-model stays web-layer-agnostic (see routes/feed.get_feed).
|
# read-model stays web-layer-agnostic (see routes/feed.get_feed).
|
||||||
|
|||||||
@@ -239,7 +239,7 @@ def deliver(
|
|||||||
# already open on the tracker and this is what would have recorded its link. We
|
# already open on the tracker and this is what would have recorded its link. We
|
||||||
# still must not write — another send owns the row and is opening its own issue —
|
# still must not write — another send owns the row and is opening its own issue —
|
||||||
# so say which issue was left behind, because nothing else will.
|
# so say which issue was left behind, because nothing else will.
|
||||||
# noqa TRY400: the traceback is one SendClaimLost carrying the session id and
|
# TRY400: the traceback is one SendClaimLost carrying the session id and
|
||||||
# nothing else; the issue number below is the whole news.
|
# nothing else; the issue number below is the whole news.
|
||||||
log.error( # noqa: TRY400
|
log.error( # noqa: TRY400
|
||||||
"an issue was created by a send that had already lost its claim",
|
"an issue was created by a send that had already lost its claim",
|
||||||
|
|||||||
+1
-1
@@ -166,7 +166,7 @@ def bind(**fields: Any):
|
|||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
for key, token in zip(keys, tokens):
|
for key, token in zip(keys, tokens, strict=True):
|
||||||
_CONTEXT_VARS[key].reset(token)
|
_CONTEXT_VARS[key].reset(token)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+37
-25
@@ -1,37 +1,36 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import math
|
import math
|
||||||
|
import mimetypes
|
||||||
from contextlib import asynccontextmanager, suppress
|
from contextlib import asynccontextmanager, suppress
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from fastapi import FastAPI, HTTPException, Request
|
from fastapi import FastAPI, HTTPException, Request
|
||||||
from fastapi.encoders import jsonable_encoder
|
from fastapi.encoders import jsonable_encoder
|
||||||
from fastapi.exceptions import RequestValidationError
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
|
||||||
# Logging is configured HERE, before the route modules are imported, so every logger they create at
|
|
||||||
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
|
||||||
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
|
||||||
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
|
||||||
from app import logs
|
|
||||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
|
||||||
|
|
||||||
configure_logging("api")
|
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
from sqlalchemy.exc import DataError
|
from sqlalchemy.exc import DataError
|
||||||
from starlette.middleware.sessions import SessionMiddleware
|
from starlette.middleware.sessions import SessionMiddleware
|
||||||
|
|
||||||
from app import bodylimit
|
# Logging is configured HERE, before the app's own modules are imported, so a line one of them logs
|
||||||
from app import logsink
|
# WHILE importing already has the format (a logger is configured whenever it is created —
|
||||||
from app import state
|
# app/logs.py — but a line emitted before `configure_logging` goes out raw). ONE format for the api,
|
||||||
from app import sysconfig
|
# the worker, the download child and alembic alike — this replaces the old uvicorn --log-config +
|
||||||
from app.config import settings
|
# the hand-rolled fallback that used to live here, and uvicorn's access log is replaced by the
|
||||||
from app.db import SessionLocal
|
# `http.request` middleware below. The imports after it are E402 on purpose, each one marked.
|
||||||
from app.plex import stream as plex_stream
|
from app import logs
|
||||||
from app.plex import subtitles as plex_subtitles
|
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||||
from app.routes import (
|
|
||||||
|
configure_logging("api")
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first
|
||||||
|
from app.config import settings # noqa: E402 — logging is configured first
|
||||||
|
from app.db import SessionLocal, engine # noqa: E402 — logging is configured first
|
||||||
|
from app.plex import stream as plex_stream # noqa: E402 — logging is configured first
|
||||||
|
from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first
|
||||||
|
from app.routes import ( # noqa: E402 — logging is configured first
|
||||||
admin,
|
admin,
|
||||||
admin_logs as admin_logs_routes,
|
admin_logs as admin_logs_routes,
|
||||||
audit as audit_routes,
|
audit as audit_routes,
|
||||||
@@ -59,7 +58,7 @@ from app.routes import (
|
|||||||
tags,
|
tags,
|
||||||
version,
|
version,
|
||||||
)
|
)
|
||||||
from app.scheduler import shutdown_scheduler, start_scheduler
|
from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 — logging is configured first
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
@@ -76,7 +75,7 @@ async def lifespan(app: FastAPI):
|
|||||||
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
||||||
try:
|
try:
|
||||||
set_level(sysconfig.get_str(db, "log_level"))
|
set_level(sysconfig.get_str(db, "log_level"))
|
||||||
except Exception: # noqa: BLE001 — any DB/config failure keeps the bootstrap level
|
except Exception: # any DB/config failure keeps the bootstrap level
|
||||||
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
||||||
# and `%s` of the exception does not say which of them raised.
|
# and `%s` of the exception does not say which of them raised.
|
||||||
log.exception("could not read the stored log level; staying at %s", current_level())
|
log.exception("could not read the stored log level; staying at %s", current_level())
|
||||||
@@ -97,6 +96,9 @@ async def lifespan(app: FastAPI):
|
|||||||
# owns, so racing it against early playback is safe.
|
# owns, so racing it against early playback is safe.
|
||||||
sweep = asyncio.create_task(_sweep_hls_orphans())
|
sweep = asyncio.create_task(_sweep_hls_orphans())
|
||||||
reaper = asyncio.create_task(_reap_hls_sessions())
|
reaper = asyncio.create_task(_reap_hls_sessions())
|
||||||
|
# Fire and forget as well: a rebuild after an ICU upgrade takes seconds, and the title sorts serve
|
||||||
|
# meanwhile (CONCURRENTLY) in the old index's order, as they did before the restart.
|
||||||
|
heal = asyncio.create_task(_heal_collation_drift())
|
||||||
start_scheduler()
|
start_scheduler()
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
@@ -106,7 +108,7 @@ async def lifespan(app: FastAPI):
|
|||||||
# pending ("Task was destroyed but it is pending!" — noise that reads like a bug during
|
# pending ("Task was destroyed but it is pending!" — noise that reads like a bug during
|
||||||
# triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own
|
# triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own
|
||||||
# bookkeeping honest.
|
# bookkeeping honest.
|
||||||
for task in (sweep, reaper):
|
for task in (sweep, reaper, heal):
|
||||||
task.cancel()
|
task.cancel()
|
||||||
with suppress(asyncio.CancelledError):
|
with suppress(asyncio.CancelledError):
|
||||||
await task
|
await task
|
||||||
@@ -139,6 +141,18 @@ async def _reap_hls_sessions() -> None:
|
|||||||
log.exception("HLS session reap failed")
|
log.exception("HLS session reap failed")
|
||||||
|
|
||||||
|
|
||||||
|
async def _heal_collation_drift() -> None:
|
||||||
|
"""Background startup check of the ICU collation the title-sort indexes are built in (see
|
||||||
|
`collation.heal_drift`). A failure is logged, never fatal: the sorts still answer, only possibly in
|
||||||
|
the old order."""
|
||||||
|
try:
|
||||||
|
await asyncio.to_thread(collation.heal_drift, engine)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
raise
|
||||||
|
except Exception:
|
||||||
|
log.exception("ICU collation drift check failed")
|
||||||
|
|
||||||
|
|
||||||
async def _sweep_hls_orphans() -> None:
|
async def _sweep_hls_orphans() -> None:
|
||||||
"""Background half of the startup HLS sweep (see lifespan). Owns its own errors: a failure here
|
"""Background half of the startup HLS sweep (see lifespan). Owns its own errors: a failure here
|
||||||
must never take the app down, and a cancelled shutdown is not one."""
|
must never take the app down, and a cancelled shutdown is not one."""
|
||||||
@@ -316,8 +330,6 @@ app.include_router(setup_routes.router)
|
|||||||
|
|
||||||
# Ensure modern image types resolve to the right Content-Type when FileResponse guesses from the
|
# Ensure modern image types resolve to the right Content-Type when FileResponse guesses from the
|
||||||
# filename (the runtime's mimetypes db doesn't always know .webp → it'd fall back to octet-stream).
|
# filename (the runtime's mimetypes db doesn't always know .webp → it'd fall back to octet-stream).
|
||||||
import mimetypes
|
|
||||||
|
|
||||||
mimetypes.add_type("image/webp", ".webp")
|
mimetypes.add_type("image/webp", ".webp")
|
||||||
mimetypes.add_type("image/avif", ".avif")
|
mimetypes.add_type("image/avif", ".avif")
|
||||||
|
|
||||||
|
|||||||
+20
-10
@@ -23,6 +23,7 @@ from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
|||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||||
|
|
||||||
from app.db import Base
|
from app.db import Base
|
||||||
|
from app.utils import canonical_email
|
||||||
|
|
||||||
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
||||||
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
||||||
@@ -51,9 +52,6 @@ def display_text(raw: str | None) -> str | None:
|
|||||||
return (raw or "").strip()[:DISPLAY_TEXT_MAX] or None
|
return (raw or "").strip()[:DISPLAY_TEXT_MAX] or None
|
||||||
|
|
||||||
|
|
||||||
from app.utils import canonical_email
|
|
||||||
|
|
||||||
|
|
||||||
# The resume-position columns (VideoState / PlexState.position_seconds) are int4. The checkpoint
|
# The resume-position columns (VideoState / PlexState.position_seconds) are int4. The checkpoint
|
||||||
# routes bound their bodies by this, so an out-of-range position is a 422 rather than a 500.
|
# routes bound their bodies by this, so an out-of-range position is a 422 rather than a 500.
|
||||||
POSITION_MAX_S = 2**31 - 1
|
POSITION_MAX_S = 2**31 - 1
|
||||||
@@ -330,10 +328,12 @@ class Video(Base, TimestampMixin):
|
|||||||
__tablename__ = "videos"
|
__tablename__ = "videos"
|
||||||
|
|
||||||
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
||||||
# No standalone channel_id index: the composite `ix_videos_channel_pub_desc`
|
# No standalone channel_id index: the composite `ix_videos_channel_pub_cover`
|
||||||
# (channel_id, published_at DESC NULLS LAST, id) from migration 0067 leads with channel_id, so it
|
# (channel_id, published_at DESC NULLS LAST, id) from migration 0095 leads with channel_id, so it
|
||||||
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) AND the per-channel feed
|
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) and the per-channel feed
|
||||||
# sort. Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
# sort. Its INCLUDEd columns make the channel manager's sums and the feed sidebar's counts Index Only
|
||||||
|
# Scans (no heap): a column those queries start reading belongs in it (test_channel_list_index_only).
|
||||||
|
# Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
||||||
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||||
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
||||||
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
||||||
@@ -347,6 +347,9 @@ class Video(Base, TimestampMixin):
|
|||||||
DateTime(timezone=True), index=True, nullable=False, server_default=func.now()
|
DateTime(timezone=True), index=True, nullable=False, server_default=func.now()
|
||||||
)
|
)
|
||||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||||
|
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
|
||||||
|
# ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to
|
||||||
|
# `feedquery.order_by` by test_feed_sort_indexes.
|
||||||
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||||
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||||
@@ -1224,6 +1227,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
||||||
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
||||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||||
|
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline (E30)
|
||||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
search_vector: Mapped[object | None] = mapped_column(
|
search_vector: Mapped[object | None] = mapped_column(
|
||||||
@@ -1385,6 +1389,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
||||||
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
||||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||||
|
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline (E30)
|
||||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||||
search_vector: Mapped[object | None] = mapped_column(
|
search_vector: Mapped[object | None] = mapped_column(
|
||||||
@@ -1415,7 +1420,8 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
top few cast. Rows whose name no longer appears in ANY item's cast_names/directors (a library
|
top few cast. Rows whose name no longer appears in ANY item's cast_names/directors (a library
|
||||||
removal orphaned them) are pruned by the maintenance sweep, mirroring how Plex drops orphaned
|
removal orphaned them) are pruned by the maintenance sweep, mirroring how Plex drops orphaned
|
||||||
metadata. The filterable cast lives in plex_items/plex_shows.cast_names (self-cleaning with the
|
metadata. The filterable cast lives in plex_items/plex_shows.cast_names (self-cleaning with the
|
||||||
row); this table is ONLY the headshot cache."""
|
row); this table is the people's own facts: the headshot link, the TMDB link and the TMDB bio
|
||||||
|
(the headshot FILE lives in the image cache, filled by the people_enrich job)."""
|
||||||
|
|
||||||
__tablename__ = "plex_people"
|
__tablename__ = "plex_people"
|
||||||
|
|
||||||
@@ -1425,11 +1431,15 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
|||||||
# TMDB person id, linked by name from a title's TMDB credits (E5 S2 lays the foundation; the /person
|
# TMDB person id, linked by name from a title's TMDB credits (E5 S2 lays the foundation; the /person
|
||||||
# biography fetch + person-page render land in S6). NULL until a TMDB-enriched title names this person.
|
# biography fetch + person-page render land in S6). NULL until a TMDB-enriched title names this person.
|
||||||
tmdb_person_id: Mapped[int | None] = mapped_column(Integer)
|
tmdb_person_id: Mapped[int | None] = mapped_column(Integer)
|
||||||
# --- TMDB person enrichment (E5 S6): fetched from /person/{tmdb_person_id} the first time the person
|
# --- TMDB person enrichment (E5 S6): fetched from /person/{tmdb_person_id} by the people_enrich job
|
||||||
# page is opened, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
# (E30 S6), or by the first person-page open the job hasn't reached, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||||
# no-op fetch so it isn't retried every open). `known_for` = top combined-credits [{tmdb_id,
|
# no-op fetch so it isn't retried every open). `known_for` = top combined-credits [{tmdb_id,
|
||||||
# media_type, title}], resolved to library rows by guid at serve time (like recommendations). ---
|
# media_type, title}], resolved to library rows by guid at serve time (like recommendations). ---
|
||||||
biography: Mapped[str | None] = mapped_column(Text)
|
biography: Mapped[str | None] = mapped_column(Text)
|
||||||
|
# TMDB's Hungarian biography (E30 S4), from the same request's `translations`; a Hungarian reader is
|
||||||
|
# served it, and the instance-language `biography` when TMDB has none. Birthplace, aliases and
|
||||||
|
# keywords have no Hungarian on TMDB, so they stay in the instance language.
|
||||||
|
biography_hu: Mapped[str | None] = mapped_column(Text)
|
||||||
birthday: Mapped[str | None] = mapped_column(String(10)) # "YYYY-MM-DD", served as-is
|
birthday: Mapped[str | None] = mapped_column(String(10)) # "YYYY-MM-DD", served as-is
|
||||||
deathday: Mapped[str | None] = mapped_column(String(10))
|
deathday: Mapped[str | None] = mapped_column(String(10))
|
||||||
place_of_birth: Mapped[str | None] = mapped_column(Text)
|
place_of_birth: Mapped[str | None] = mapped_column(Text)
|
||||||
|
|||||||
+32
-14
@@ -15,20 +15,35 @@ def _art_url(rating_key: str, art_key: str | None) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
# Card hover-overview: a short synopsis for the reveal panel, trimmed so a strip/related row of cards
|
# Card hover-overview: a short synopsis for the reveal panel, trimmed so a strip/related row of cards
|
||||||
# stays a few hundred bytes each rather than carrying full synopses. (These non-grid card paths serve
|
# stays a few hundred bytes each rather than carrying full synopses.
|
||||||
# the EN summary/rating; the main grid coalesces the HU twin in browse.unified_library.)
|
|
||||||
def _card_summary(summary: str | None) -> str | None:
|
def _card_summary(summary: str | None) -> str | None:
|
||||||
return (summary or "")[:300] or None
|
return (summary or "")[:300] or None
|
||||||
|
|
||||||
|
|
||||||
def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
def norm_lang(lang: str | None) -> str:
|
||||||
|
"""The overlay has an EN base + an optional HU twin; anything other than `hu` serves EN."""
|
||||||
|
return "hu" if lang == "hu" else "en"
|
||||||
|
|
||||||
|
|
||||||
|
# Every card speaks the reader's language (E30): `lang="hu"` serves the HU overlay twin's title and
|
||||||
|
# summary, falling back to EN where a row has none — the same rule as browse.unified_library's
|
||||||
|
# `title_col`/`summary_col`. These rails once served English to a Hungarian reader.
|
||||||
|
def display_title(row: PlexItem | PlexShow, lang: str) -> str:
|
||||||
|
return (row.title_hu or row.title) if lang == "hu" else row.title
|
||||||
|
|
||||||
|
|
||||||
|
def display_summary(row: PlexItem | PlexShow, lang: str) -> str | None:
|
||||||
|
return (row.summary_hu or row.summary) if lang == "hu" else row.summary
|
||||||
|
|
||||||
|
|
||||||
|
def movie_card(it: PlexItem, st: PlexState | None, *, lang: str) -> dict:
|
||||||
return {
|
return {
|
||||||
"id": it.rating_key,
|
"id": it.rating_key,
|
||||||
"type": "movie",
|
"type": "movie",
|
||||||
"title": it.title,
|
"title": display_title(it, lang),
|
||||||
"year": it.year,
|
"year": it.year,
|
||||||
"rating": it.rating,
|
"rating": it.rating,
|
||||||
"summary": _card_summary(it.summary),
|
"summary": _card_summary(display_summary(it, lang)),
|
||||||
"duration_seconds": it.duration_s,
|
"duration_seconds": it.duration_s,
|
||||||
"thumb": f"/api/plex/image/{it.rating_key}",
|
"thumb": f"/api/plex/image/{it.rating_key}",
|
||||||
"art": _art_url(it.rating_key, it.art_key),
|
"art": _art_url(it.rating_key, it.art_key),
|
||||||
@@ -39,14 +54,14 @@ def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def show_card(sh: PlexShow, status: str = "new") -> dict:
|
def show_card(sh: PlexShow, status: str = "new", *, lang: str) -> dict:
|
||||||
return {
|
return {
|
||||||
"id": sh.rating_key,
|
"id": sh.rating_key,
|
||||||
"type": "show",
|
"type": "show",
|
||||||
"title": sh.title,
|
"title": display_title(sh, lang),
|
||||||
"year": sh.year,
|
"year": sh.year,
|
||||||
"rating": sh.rating,
|
"rating": sh.rating,
|
||||||
"summary": _card_summary(sh.summary),
|
"summary": _card_summary(display_summary(sh, lang)),
|
||||||
"thumb": f"/api/plex/image/{sh.rating_key}",
|
"thumb": f"/api/plex/image/{sh.rating_key}",
|
||||||
"art": _art_url(sh.rating_key, sh.art_key),
|
"art": _art_url(sh.rating_key, sh.art_key),
|
||||||
"season_count": sh.child_count,
|
"season_count": sh.child_count,
|
||||||
@@ -55,12 +70,12 @@ def show_card(sh: PlexShow, status: str = "new") -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def episode_card(e: PlexItem, st: PlexState | None) -> dict:
|
def episode_card(e: PlexItem, st: PlexState | None, *, lang: str) -> dict:
|
||||||
return {
|
return {
|
||||||
"id": e.rating_key,
|
"id": e.rating_key,
|
||||||
"type": "episode",
|
"type": "episode",
|
||||||
"title": e.title,
|
"title": display_title(e, lang),
|
||||||
"summary": e.summary,
|
"summary": display_summary(e, lang),
|
||||||
"season_number": e.season_number,
|
"season_number": e.season_number,
|
||||||
"episode_number": e.episode_number,
|
"episode_number": e.episode_number,
|
||||||
"duration_seconds": e.duration_s,
|
"duration_seconds": e.duration_s,
|
||||||
@@ -77,12 +92,15 @@ def leaf_card(
|
|||||||
st: PlexState | None,
|
st: PlexState | None,
|
||||||
show_title: str | None = None,
|
show_title: str | None = None,
|
||||||
show_rk: str | None = None,
|
show_rk: str | None = None,
|
||||||
|
*,
|
||||||
|
lang: str,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""A playable-leaf card (movie or episode). Used by playlists, which can mix both kinds.
|
"""A playable-leaf card (movie or episode). Used by playlists, which can mix both kinds.
|
||||||
`show_rk` (the show's rating_key) lets the client group a playlist's episodes by show/season."""
|
`show_rk` (the show's rating_key) lets the client group a playlist's episodes by show/season.
|
||||||
|
`show_title` is the caller's, already in `lang` (see `display_title`)."""
|
||||||
if it.kind == "episode":
|
if it.kind == "episode":
|
||||||
card = episode_card(it, st)
|
card = episode_card(it, st, lang=lang)
|
||||||
card["show_title"] = show_title
|
card["show_title"] = show_title
|
||||||
card["show_id"] = show_rk
|
card["show_id"] = show_rk
|
||||||
return card
|
return card
|
||||||
return movie_card(it, st)
|
return movie_card(it, st, lang=lang)
|
||||||
|
|||||||
@@ -78,9 +78,10 @@ class PlexClient:
|
|||||||
# --- Catalog mirror (P1) ---
|
# --- Catalog mirror (P1) ---
|
||||||
def section_items(
|
def section_items(
|
||||||
self, key: str, item_type: int, start: int = 0, size: int = 200
|
self, key: str, item_type: int, start: int = 0, size: int = 200
|
||||||
) -> tuple[list[dict], int]:
|
) -> tuple[list[dict], int | None]:
|
||||||
"""A page of a section's top-level items. type 1 = movie, 2 = show. Returns
|
"""A page of a section's top-level items. type 1 = movie, 2 = show. Returns
|
||||||
(items, total_size)."""
|
(items, total_size); the total is None when the answer carries none, so a container that
|
||||||
|
says "0" (a genuinely empty section) stays distinguishable from one that says nothing."""
|
||||||
mc = self._get(
|
mc = self._get(
|
||||||
f"/library/sections/{key}/all",
|
f"/library/sections/{key}/all",
|
||||||
params={
|
params={
|
||||||
@@ -93,7 +94,10 @@ class PlexClient:
|
|||||||
"X-Plex-Container-Size": size,
|
"X-Plex-Container-Size": size,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
return mc.get("Metadata", []) or [], int(mc.get("totalSize", mc.get("size", 0)) or 0)
|
# `totalSize` only: `size` is THIS page's count, and read as the total it made a single
|
||||||
|
# page look like the whole section — "complete" to a prune or a reconcile (E32 review).
|
||||||
|
total = mc.get("totalSize")
|
||||||
|
return mc.get("Metadata", []) or [], None if total is None else int(total)
|
||||||
|
|
||||||
def children(self, rating_key: str) -> list[dict]:
|
def children(self, rating_key: str) -> list[dict]:
|
||||||
"""Direct children (show → seasons, season → episodes)."""
|
"""Direct children (show → seasons, season → episodes)."""
|
||||||
@@ -136,9 +140,14 @@ class PlexClient:
|
|||||||
# --- Collection editing (P2 — write-back to Plex; all shared/global, admin-gated in the routes) ---
|
# --- Collection editing (P2 — write-back to Plex; all shared/global, admin-gated in the routes) ---
|
||||||
@property
|
@property
|
||||||
def machine_id(self) -> str:
|
def machine_id(self) -> str:
|
||||||
"""Server machineIdentifier — needed in the `uri` of collection add/create calls. Cached."""
|
"""Server machineIdentifier — needed in the `uri` of collection add/create calls. Cached.
|
||||||
|
Without one the uri would read `server:///…`, a malformed write Plex answers in its own way;
|
||||||
|
it is refused here, by name (E32 S2)."""
|
||||||
if not getattr(self, "_machine_id", None):
|
if not getattr(self, "_machine_id", None):
|
||||||
self._machine_id = self._get("/identity").get("machineIdentifier") or ""
|
mid = self._get("/identity").get("machineIdentifier")
|
||||||
|
if not mid:
|
||||||
|
raise PlexError("Plex did not report its machine identifier")
|
||||||
|
self._machine_id = mid
|
||||||
return self._machine_id
|
return self._machine_id
|
||||||
|
|
||||||
def _write(self, method: str, path: str) -> dict:
|
def _write(self, method: str, path: str) -> dict:
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
"""One Hungarian word per Plex genre (E30).
|
||||||
|
|
||||||
|
Plex's Hungarian agent returns a MIXED vocabulary: some genres come back translated ("Dráma",
|
||||||
|
"Vígjáték"), others stay English ("Action", "Adventure", "Biography"), and a title with no HU twin has
|
||||||
|
only its English genres. The HU genre list therefore offered "Drama" beside "Dráma" and a dozen English
|
||||||
|
words. This map sends every genre word we have met — English (Plex EN agent, TMDB, IMDb) or Plex's
|
||||||
|
Hungarian — to the ONE word a Hungarian reader sees. The overlay sync writes it into `genres_hu`, and a
|
||||||
|
genre filter is read through it too, so a saved filter still holding "Action" keeps matching "Akció".
|
||||||
|
|
||||||
|
A word not in the map passes through unchanged: an unknown genre stays visible (in whatever language
|
||||||
|
Plex gave it) rather than disappearing.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
|
# Keys are matched case-insensitively (see `_BY_KEY`). The Hungarian values double as keys mapping to
|
||||||
|
# themselves, so running a list through twice changes nothing.
|
||||||
|
_HU: dict[str, str] = {
|
||||||
|
"Action": "Akció",
|
||||||
|
"Action & Adventure": "Akció és kaland",
|
||||||
|
"Adventure": "Kaland",
|
||||||
|
"Animation": "Animációs",
|
||||||
|
"Anime": "Anime",
|
||||||
|
"Biography": "Életrajzi",
|
||||||
|
"Children": "Gyerek",
|
||||||
|
"Comedy": "Vígjáték",
|
||||||
|
"Crime": "Bűnügyi",
|
||||||
|
"Documentary": "Dokumentum",
|
||||||
|
"Drama": "Dráma",
|
||||||
|
"Family": "Családi",
|
||||||
|
"Fantasy": "Fantasy",
|
||||||
|
"Film-Noir": "Film noir",
|
||||||
|
"Food": "Gasztronómia",
|
||||||
|
"Game Show": "Vetélkedő",
|
||||||
|
"Game-Show": "Vetélkedő",
|
||||||
|
"History": "Történelmi",
|
||||||
|
"Home and Garden": "Otthon és kert",
|
||||||
|
"Horror": "Horror",
|
||||||
|
"Indie": "Független",
|
||||||
|
"Kids": "Gyerek",
|
||||||
|
"Martial Arts": "Harcművészet",
|
||||||
|
"Mini-Series": "Minisorozat",
|
||||||
|
"Music": "Zenei",
|
||||||
|
"Musical": "Musical",
|
||||||
|
"Mystery": "Rejtély",
|
||||||
|
"News": "Hírek",
|
||||||
|
"Reality": "Valóságshow",
|
||||||
|
"Reality-TV": "Valóságshow",
|
||||||
|
"Romance": "Romantikus",
|
||||||
|
"Sci-Fi": "Sci-Fi",
|
||||||
|
"Sci-Fi & Fantasy": "Sci-Fi és fantasy",
|
||||||
|
"Science Fiction": "Sci-Fi",
|
||||||
|
"Short": "Rövidfilm",
|
||||||
|
"Soap": "Szappanopera",
|
||||||
|
"Sport": "Sport",
|
||||||
|
"Suspense": "Feszültség",
|
||||||
|
"Talk": "Talkshow",
|
||||||
|
"Talk Show": "Talkshow",
|
||||||
|
"Thriller": "Thriller",
|
||||||
|
"Travel": "Utazás",
|
||||||
|
"TV Movie": "TV film",
|
||||||
|
"War": "Háborús",
|
||||||
|
"War & Politics": "Háború és politika",
|
||||||
|
"Western": "Western",
|
||||||
|
# Plex's own Hungarian words, pinned to themselves (and to the spelling above where they differ).
|
||||||
|
"Akció": "Akció",
|
||||||
|
"Animációs": "Animációs",
|
||||||
|
"Bűnügyi": "Bűnügyi",
|
||||||
|
"Családi": "Családi",
|
||||||
|
"Dokumentum": "Dokumentum",
|
||||||
|
"Dokumentumfilm": "Dokumentum",
|
||||||
|
"Dráma": "Dráma",
|
||||||
|
"Életrajzi": "Életrajzi",
|
||||||
|
"Háborús": "Háborús",
|
||||||
|
"Kaland": "Kaland",
|
||||||
|
"Rejtély": "Rejtély",
|
||||||
|
"Romantikus": "Romantikus",
|
||||||
|
"Történelmi": "Történelmi",
|
||||||
|
"TV film": "TV film",
|
||||||
|
"Vígjáték": "Vígjáték",
|
||||||
|
"Zene": "Zenei",
|
||||||
|
"Zenei": "Zenei",
|
||||||
|
}
|
||||||
|
_BY_KEY = {k.casefold(): v for k, v in _HU.items()}
|
||||||
|
|
||||||
|
|
||||||
|
def hu_genre(word: str) -> str:
|
||||||
|
"""The Hungarian word for one genre; an unknown word comes back unchanged."""
|
||||||
|
return _BY_KEY.get(word.strip().casefold(), word)
|
||||||
|
|
||||||
|
|
||||||
|
def hu_genres(words: Iterable[str] | None) -> list[str] | None:
|
||||||
|
"""A genre list in Hungarian, in its first-seen order, each word once ("Drama" + "Dráma" → one
|
||||||
|
"Dráma"). None or empty stays None, like the listing's own `_tags`."""
|
||||||
|
out: list[str] = []
|
||||||
|
for w in words or ():
|
||||||
|
hu = hu_genre(w)
|
||||||
|
if hu not in out:
|
||||||
|
out.append(hu)
|
||||||
|
return out or None
|
||||||
@@ -8,6 +8,9 @@ few hundred MB at most for a big library — acceptable, matching the download d
|
|||||||
The cache directory (``cache.CACHE_DIR``) is shared with the subtitle cache (see ``subtitles.py``).
|
The cache directory (``cache.CACHE_DIR``) is shared with the subtitle cache (see ``subtitles.py``).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import time
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
from fastapi import Response
|
from fastapi import Response
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
@@ -20,9 +23,23 @@ log = logs.get_logger(__name__)
|
|||||||
|
|
||||||
_EXT_CT = [("jpg", "image/jpeg"), ("png", "image/png"), ("webp", "image/webp")]
|
_EXT_CT = [("jpg", "image/jpeg"), ("png", "image/png"), ("webp", "image/webp")]
|
||||||
_CT_EXT = {ct: ext for ext, ct in _EXT_CT}
|
_CT_EXT = {ct: ext for ext, ct in _EXT_CT}
|
||||||
# Target box per variant — Plex resizes to these server-side. Posters fill ≤176px cells (grid/info/
|
# Target box per variant — Plex resizes to these server-side. The full poster (400px) covers a detail
|
||||||
# strips), so 400px covers 2× DPR; the faint backdrop art only needs ~1280px. Keeps images ~tens of KB.
|
# page's ≤224px poster at 2× DPR, the full art (1280px) a hero backdrop. A card is much smaller: a grid
|
||||||
SIZES = {"thumb": (400, 600), "art": (1280, 720)}
|
# poster is ~144px and a shelf's landscape art ~212px, so the `_sm` pair serves those at 1.33× DPR
|
||||||
|
# (E31 S4e: cards fetched the 400 poster, 42 KB, and the 1280 art for a 212px picture). The browser
|
||||||
|
# picks between a pair itself (`srcset`, built by the frontend's `lib/plexImage` cardImage, whose
|
||||||
|
# widths plexImage.contract.test holds to these).
|
||||||
|
SIZES = {
|
||||||
|
"thumb": (400, 600),
|
||||||
|
"thumb_sm": (240, 360),
|
||||||
|
"art": (1280, 720),
|
||||||
|
"art_sm": (480, 270),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def picture(variant: str) -> str:
|
||||||
|
"""Which stored picture a size variant is cut from: the poster (`thumb`) or the backdrop (`art`)."""
|
||||||
|
return "art" if variant.startswith("art") else "thumb"
|
||||||
|
|
||||||
|
|
||||||
def key_for(db: Session, rating_key: str, variant: str) -> str | None:
|
def key_for(db: Session, rating_key: str, variant: str) -> str | None:
|
||||||
@@ -56,6 +73,49 @@ def read(key: str) -> tuple[bytes, str] | None:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def has(key: str) -> bool:
|
||||||
|
"""A cached, non-empty file exists for `key` (a prefetch's cheap check — no read)."""
|
||||||
|
for ext, _ in _EXT_CT:
|
||||||
|
try:
|
||||||
|
if (CACHE_DIR / f"{key}.{ext}").stat().st_size > 0:
|
||||||
|
return True
|
||||||
|
# silent-ok: a miss, as in read().
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def drop(key: str) -> None:
|
||||||
|
"""Remove `key`'s file, whatever its extension (a superseded variant). Best-effort."""
|
||||||
|
for ext, _ in _EXT_CT:
|
||||||
|
try:
|
||||||
|
(CACHE_DIR / f"{key}.{ext}").unlink(missing_ok=True)
|
||||||
|
# silent-ok: a leftover original only costs disk; the small twin is already written.
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def mark_miss(key: str) -> None:
|
||||||
|
"""Remember that `key`'s source could not be fetched (an empty `.miss` file; its mtime is when), so
|
||||||
|
a prefetch can leave a dead source alone for a while. Best-effort."""
|
||||||
|
try:
|
||||||
|
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
(CACHE_DIR / f"{key}.miss").touch()
|
||||||
|
# silent-ok: without the mark the dead source is only asked for again on the next run.
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def missed_within(key: str, age: timedelta) -> bool:
|
||||||
|
"""`key` was marked as a miss less than `age` ago."""
|
||||||
|
try:
|
||||||
|
mtime = (CACHE_DIR / f"{key}.miss").stat().st_mtime
|
||||||
|
# silent-ok: no mark (the usual case).
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
return time.time() - mtime < age.total_seconds()
|
||||||
|
|
||||||
|
|
||||||
def write(key: str, data: bytes, ct: str) -> None:
|
def write(key: str, data: bytes, ct: str) -> None:
|
||||||
ext = _CT_EXT.get((ct or "").split(";")[0].strip(), "jpg")
|
ext = _CT_EXT.get((ct or "").split(";")[0].strip(), "jpg")
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
"""The HU overlay: a Hungarian Plex section written onto its English twin's rows (E30), split out of
|
||||||
|
`plex/sync.py` (E32 S2), which runs it once per paired section during the full sync.
|
||||||
|
|
||||||
|
The pairing is admin-configured (`plex_overlay_map`, ``huKey:enKey,...``). An overlay section creates
|
||||||
|
no browsable rows of its own; it only fills the `*_hu` columns of the EN rows it matches by guid.
|
||||||
|
`plex/sync.py` and this module import each other as MODULES (attribute access at call time), which is
|
||||||
|
what lets either be imported first.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import logs, sysconfig
|
||||||
|
from app.models import PlexItem, PlexLibrary, PlexShow
|
||||||
|
from app.plex import sync as plex_sync
|
||||||
|
from app.plex.client import PlexClient
|
||||||
|
from app.plex.genres import hu_genres
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def overlay_map(db: Session) -> dict[str, str]:
|
||||||
|
"""The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
|
||||||
|
('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
|
||||||
|
Malformed pairs are skipped."""
|
||||||
|
raw = sysconfig.get_str(db, "plex_overlay_map")
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
for pair in raw.split(","):
|
||||||
|
hu, sep, en = pair.partition(":")
|
||||||
|
hu, en = hu.strip(), en.strip()
|
||||||
|
if sep and hu and en:
|
||||||
|
out[hu] = en
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def sync_overlay(
|
||||||
|
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
||||||
|
) -> None:
|
||||||
|
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
||||||
|
NO browsable rows — an overlay library is never listed on its own. Only title/summary/tagline/genres
|
||||||
|
differ by language; the language-neutral people_text feeds both search vectors. A show section's
|
||||||
|
episodes are overlaid too (E30): the HU section lists every episode with its Hungarian title and
|
||||||
|
summary, and the episode rows live in the EN show library.
|
||||||
|
|
||||||
|
This is a full RECONCILE: after a COMPLETE HU listing every guid-bearing EN row's HU fields are
|
||||||
|
cleared, then the matches are repopulated — so an EN row whose HU twin vanished (or was re-guid'd)
|
||||||
|
falls back to EN instead of keeping stale Hungarian metadata. A listing cut short only adds. HU
|
||||||
|
items with no EN match are skipped."""
|
||||||
|
if lib.kind == "movie":
|
||||||
|
passes = [
|
||||||
|
(
|
||||||
|
plex_sync._TYPE_MOVIE,
|
||||||
|
select(PlexItem).where(
|
||||||
|
PlexItem.library_id == en_lib_id,
|
||||||
|
PlexItem.kind == "movie",
|
||||||
|
PlexItem.guid.is_not(None),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
passes = [
|
||||||
|
(
|
||||||
|
plex_sync._TYPE_SHOW,
|
||||||
|
select(PlexShow).where(
|
||||||
|
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
plex_sync._TYPE_EPISODE,
|
||||||
|
select(PlexItem).where(
|
||||||
|
PlexItem.library_id == en_lib_id,
|
||||||
|
PlexItem.kind == "episode",
|
||||||
|
PlexItem.guid.is_not(None),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for item_type, rows_q in passes:
|
||||||
|
_overlay_pass(db, plex, lib, item_type, db.execute(rows_q).scalars(), stats)
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _overlay_pass(
|
||||||
|
db: Session,
|
||||||
|
plex: PlexClient,
|
||||||
|
lib: PlexLibrary,
|
||||||
|
item_type: int,
|
||||||
|
rows: Iterable[PlexItem] | Iterable[PlexShow],
|
||||||
|
stats: dict,
|
||||||
|
) -> None:
|
||||||
|
"""One kind of the overlay (movies, shows or episodes) — see `sync_overlay`."""
|
||||||
|
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
||||||
|
by_guid: dict[str, list[PlexItem | PlexShow]] = {}
|
||||||
|
for r in rows:
|
||||||
|
if r.guid: # the passes select guid-bearing rows only
|
||||||
|
by_guid.setdefault(r.guid, []).append(r)
|
||||||
|
# The HU listing is read first and applied after, so the clear can wait for its verdict.
|
||||||
|
hu_by_guid, complete = _read_hu_listing(plex, lib, item_type, by_guid)
|
||||||
|
episodes = item_type == plex_sync._TYPE_EPISODE
|
||||||
|
if complete:
|
||||||
|
_clear_hu(by_guid, episodes)
|
||||||
|
else:
|
||||||
|
stats["overlay_incomplete"] = stats.get("overlay_incomplete", 0) + 1
|
||||||
|
log.warning(
|
||||||
|
"the HU listing ended before the server's total; existing HU text is kept",
|
||||||
|
extra=logs.event("plex.sync.overlay_incomplete", library=lib.plex_key),
|
||||||
|
)
|
||||||
|
for g, meta in hu_by_guid.items():
|
||||||
|
targets = by_guid[g]
|
||||||
|
title_hu = (meta.get("title") or "").strip() or None
|
||||||
|
summary_hu = meta.get("summary")
|
||||||
|
tagline_hu = (meta.get("tagline") or "").strip() or None
|
||||||
|
genres_hu = hu_genres(plex_sync._tags(meta, "Genre"))
|
||||||
|
for row in targets:
|
||||||
|
row.title_hu = title_hu
|
||||||
|
row.summary_hu = summary_hu
|
||||||
|
if not episodes:
|
||||||
|
row.tagline_hu = tagline_hu
|
||||||
|
row.genres_hu = genres_hu or row.genres_hu
|
||||||
|
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
||||||
|
|
||||||
|
|
||||||
|
def _read_hu_listing(
|
||||||
|
plex: PlexClient, lib: PlexLibrary, item_type: int, by_guid: dict
|
||||||
|
) -> tuple[dict[str, dict], bool]:
|
||||||
|
"""The HU section's entries for the guids the EN side carries, and whether the walk reached the
|
||||||
|
server's total. The first HU entry for a guid wins; a dup HU item can't overwrite or double-count."""
|
||||||
|
progress: dict = {}
|
||||||
|
hu_by_guid: dict[str, dict] = {}
|
||||||
|
for meta in plex_sync._paginate(plex, lib.plex_key, item_type, progress):
|
||||||
|
g = plex_sync._canonical_guid(meta)
|
||||||
|
if g and g in by_guid and g not in hu_by_guid:
|
||||||
|
hu_by_guid[g] = meta
|
||||||
|
return hu_by_guid, bool(progress.get("complete"))
|
||||||
|
|
||||||
|
|
||||||
|
def _clear_hu(by_guid: dict, episodes: bool) -> None:
|
||||||
|
"""Reconcile step: clear HU fields on all guid-bearing rows; the matches are re-set after. Only
|
||||||
|
rows that actually held HU data become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is
|
||||||
|
cheap. The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking
|
||||||
|
one, from its own English list — otherwise the HU genre list mixed "Drama" (no twin) with "Dráma".
|
||||||
|
Only after a COMPLETE listing: one a Plex hiccup cut short would clear the HU text of every title
|
||||||
|
it failed to reach, until the next full sync (E32 S2, the watch reconcile's twin)."""
|
||||||
|
for group in by_guid.values():
|
||||||
|
for row in group:
|
||||||
|
row.title_hu = row.summary_hu = None
|
||||||
|
if not episodes:
|
||||||
|
row.tagline_hu = None
|
||||||
|
row.genres_hu = hu_genres(row.genres)
|
||||||
+176
-10
@@ -3,13 +3,23 @@ or show (cast_names + the people_text search blob + the plex_people headshot cac
|
|||||||
orphaned people. Used by BOTH the info-page lazy enrichment (routes.plex) and the background rich
|
orphaned people. Used by BOTH the info-page lazy enrichment (routes.plex) and the background rich
|
||||||
re-sync (plex.sync), so the two paths never diverge."""
|
re-sync (plex.sync), so the two paths never diverge."""
|
||||||
|
|
||||||
from urllib.parse import quote
|
import hashlib
|
||||||
|
import re
|
||||||
|
from datetime import timedelta
|
||||||
|
from typing import Literal
|
||||||
|
from urllib.parse import quote, unquote
|
||||||
|
|
||||||
|
import httpx
|
||||||
from sqlalchemy import select, text
|
from sqlalchemy import select, text
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import logs, sysconfig
|
||||||
from app.models import PlexPerson
|
from app.models import PlexPerson
|
||||||
|
from app.plex import imgcache
|
||||||
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# Cast/crew photos come from Plex's PUBLIC metadata CDN (no token). We proxy them (host-whitelisted)
|
# Cast/crew photos come from Plex's PUBLIC metadata CDN (no token). We proxy them (host-whitelisted)
|
||||||
# so the user's browser makes no third-party request (privacy + self-host CSP friendliness).
|
# so the user's browser makes no third-party request (privacy + self-host CSP friendliness).
|
||||||
@@ -19,9 +29,7 @@ PERSON_IMG_HOST = "https://metadata-static.plex.tv/"
|
|||||||
def person_thumb_url(thumb: str | None) -> str | None:
|
def person_thumb_url(thumb: str | None) -> str | None:
|
||||||
"""Proxied /person-image URL for a Plex CDN headshot, or None when it's missing / off-CDN."""
|
"""Proxied /person-image URL for a Plex CDN headshot, or None when it's missing / off-CDN."""
|
||||||
t = str(thumb or "")
|
t = str(thumb or "")
|
||||||
return (
|
return f"/api/plex/person-image?u={quote(t, safe='')}" if is_portrait_url(t) else None
|
||||||
f"/api/plex/person-image?u={quote(t, safe='')}" if t.startswith(PERSON_IMG_HOST) else None
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def cast_from_meta(meta: dict, limit: int = 24) -> list[dict]:
|
def cast_from_meta(meta: dict, limit: int = 24) -> list[dict]:
|
||||||
@@ -89,14 +97,23 @@ def enrich_row_people(db: Session, row, cast: list[dict]) -> bool:
|
|||||||
return dirty
|
return dirty
|
||||||
|
|
||||||
|
|
||||||
|
# Every credit reference: one person name per (movie/show, credit column). The ONE list of credit
|
||||||
|
# columns, so the orphan prune and the enrichment order never disagree about who is credited.
|
||||||
|
_CREDIT_COLUMNS = [
|
||||||
|
(table, col) for table in ("plex_items", "plex_shows") for col in ("cast_names", "directors")
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _credit_refs_sql(union: Literal["UNION", "UNION ALL"]) -> str:
|
||||||
|
return f" {union} ".join(
|
||||||
|
f"SELECT jsonb_array_elements_text({col}) FROM {table} WHERE jsonb_typeof({col}) = 'array'"
|
||||||
|
for table, col in _CREDIT_COLUMNS
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside
|
# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside
|
||||||
# this set are orphans left by a library removal.
|
# this set are orphans left by a library removal.
|
||||||
_REFERENCED_PEOPLE_SQL = """
|
_REFERENCED_PEOPLE_SQL = _credit_refs_sql("UNION")
|
||||||
SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array'
|
|
||||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array'
|
|
||||||
UNION SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array'
|
|
||||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array'
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def prune_orphan_people(db: Session) -> int:
|
def prune_orphan_people(db: Session) -> int:
|
||||||
@@ -113,3 +130,152 @@ def prune_orphan_people(db: Session) -> int:
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
return res.rowcount or 0
|
return res.rowcount or 0
|
||||||
|
|
||||||
|
|
||||||
|
# How often each person is credited across movies, shows and episodes (cast OR director) — the order the
|
||||||
|
# people_enrich job works in, so the faces met most often are local first.
|
||||||
|
_CREDIT_COUNTS_SQL = (
|
||||||
|
f"SELECT ref, count(*) AS n FROM ({_credit_refs_sql('UNION ALL')}) AS refs(ref) GROUP BY ref"
|
||||||
|
)
|
||||||
|
_NEEDS = {
|
||||||
|
# linked to TMDB, bio never fetched
|
||||||
|
"bio": "p.tmdb_person_id IS NOT NULL AND p.tmdb_enriched_at IS NULL",
|
||||||
|
# has a Plex headshot (whether it is cached is the disk's answer, not the row's)
|
||||||
|
"portrait": "p.photo_thumb IS NOT NULL",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def people_by_credits(
|
||||||
|
db: Session, need: Literal["bio", "portrait"], limit: int | None
|
||||||
|
) -> list[int]:
|
||||||
|
"""plex_people ids that `need` something, the most-credited first (ties by id)."""
|
||||||
|
rows = db.execute(
|
||||||
|
text(
|
||||||
|
f"SELECT p.id FROM plex_people p LEFT JOIN ({_CREDIT_COUNTS_SQL}) c ON c.ref = p.name "
|
||||||
|
f"WHERE {_NEEDS[need]} ORDER BY coalesce(c.n, 0) DESC, p.id"
|
||||||
|
+ (" LIMIT :limit" if limit is not None else "")
|
||||||
|
),
|
||||||
|
{"limit": limit} if limit is not None else {},
|
||||||
|
)
|
||||||
|
return [r[0] for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
# --- Portraits (E30 S6) ---------------------------------------------------------------------------
|
||||||
|
# A Plex CDN headshot is a full-size original (~170 KB on average, measured). Plex's photo transcoder
|
||||||
|
# fits it into this box (~13 KB), which covers the largest place a portrait is drawn (the person page's
|
||||||
|
# 6rem avatar) at 2x density and a raised text size. The box is part of the cache key, like a poster's:
|
||||||
|
# a new box re-caches instead of serving a stale size. Only a resized image is ever written under it —
|
||||||
|
# an original fetched without Plex keeps its own key, and goes once its small twin lands.
|
||||||
|
PORTRAIT_BOX = 320
|
||||||
|
# A headshot Plex answered "not found" for (a dead CDN file) is not asked for again this long, so a
|
||||||
|
# few dead faces never hold up the rest of the prefetch.
|
||||||
|
PORTRAIT_RETRY = timedelta(days=7)
|
||||||
|
# The one shape a Plex CDN headshot has (…/7/people/<hash>.jpg): plain path segments, no query, no
|
||||||
|
# fragment, no dot segment or empty segment (an HTTP client folds `a/../b` and the CDN `a//b` onto one
|
||||||
|
# file). The person-image proxy takes nothing else, so it cannot be made to cache endless spellings of
|
||||||
|
# one file (the image cache has no eviction). Used where a stored headshot is made, served and warmed.
|
||||||
|
_SEGMENT = r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*" # a name, dots only between its parts
|
||||||
|
_PORTRAIT_URL = re.compile(re.escape(PERSON_IMG_HOST) + _SEGMENT + rf"(?:/{_SEGMENT})*")
|
||||||
|
|
||||||
|
|
||||||
|
def is_portrait_url(url: str) -> bool:
|
||||||
|
return _PORTRAIT_URL.fullmatch(url) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def _portrait_keys(url: str) -> tuple[str, str]:
|
||||||
|
"""(the resized portrait's key, the full-size original's key)."""
|
||||||
|
base = "person_" + hashlib.sha1(url.encode()).hexdigest()
|
||||||
|
return f"{base}_{PORTRAIT_BOX}", base
|
||||||
|
|
||||||
|
|
||||||
|
def portrait_cached(url: str) -> tuple[bytes, str] | None:
|
||||||
|
"""The cached portrait: the resized one, else an original kept from a time without Plex."""
|
||||||
|
small, original = _portrait_keys(url)
|
||||||
|
return imgcache.read(small) or imgcache.read(original)
|
||||||
|
|
||||||
|
|
||||||
|
def cache_portrait(plex: PlexClient | None, url: str) -> tuple[bytes, str]:
|
||||||
|
"""Fetch one CDN headshot and cache it. With a Plex client it is resized (the box key; the original
|
||||||
|
is removed); without one the original comes straight from the public CDN and is cached under its own
|
||||||
|
key, so the job still resizes it once Plex is there. Raises PlexError / httpx.HTTPError."""
|
||||||
|
small, original = _portrait_keys(url)
|
||||||
|
if plex is None:
|
||||||
|
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
||||||
|
r = c.get(url)
|
||||||
|
r.raise_for_status()
|
||||||
|
data, ct = r.content, r.headers.get("content-type", "image/jpeg")
|
||||||
|
imgcache.write(original, data, ct)
|
||||||
|
return data, ct
|
||||||
|
data, ct = plex.image_bytes(url, PORTRAIT_BOX, PORTRAIT_BOX)
|
||||||
|
imgcache.write(small, data, ct)
|
||||||
|
imgcache.drop(original)
|
||||||
|
return data, ct
|
||||||
|
|
||||||
|
|
||||||
|
def cdn_url(photo_thumb: str) -> str | None:
|
||||||
|
"""The CDN url inside a stored proxied `/person-image?u=` value (see person_thumb_url)."""
|
||||||
|
_, _, q = photo_thumb.partition("?u=")
|
||||||
|
url = unquote(q)
|
||||||
|
return url if is_portrait_url(url) else None
|
||||||
|
|
||||||
|
|
||||||
|
# Plex failing this many headshots in a row ends a run: it is Plex that is failing, not the files.
|
||||||
|
_PLEX_DOWN_AFTER = 3
|
||||||
|
# The statuses that mean the one file is gone.
|
||||||
|
_DEAD = frozenset({404, 410})
|
||||||
|
|
||||||
|
|
||||||
|
def warm_portraits(db: Session, limit: int) -> dict:
|
||||||
|
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first (E30 S6), so a
|
||||||
|
person's face comes from local disk. Needs Plex (the resize) and the Plex module on; without them
|
||||||
|
nothing is prefetched — full-size originals for every person would cost gigabytes. A headshot Plex
|
||||||
|
answers "not found" for is marked and left for PORTRAIT_RETRY; Plex failing in a row ends the run."""
|
||||||
|
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||||
|
return {"cached": 0, "skipped": "disabled"}
|
||||||
|
try:
|
||||||
|
plex = PlexClient(db)
|
||||||
|
# silent-ok: no Plex server set up is a setting, not a failure; the run's result says "disabled".
|
||||||
|
except PlexNotConfigured:
|
||||||
|
return {"cached": 0, "skipped": "disabled"}
|
||||||
|
photos: dict[int, str] = {
|
||||||
|
pid: thumb
|
||||||
|
for pid, thumb in db.execute(
|
||||||
|
select(PlexPerson.id, PlexPerson.photo_thumb).where(PlexPerson.photo_thumb.is_not(None))
|
||||||
|
).tuples()
|
||||||
|
if thumb
|
||||||
|
}
|
||||||
|
# `down` counts Plex failing in a row (any answer from Plex breaks it), `plex_failed` in all
|
||||||
|
done = dead = plex_failed = down = 0
|
||||||
|
with plex:
|
||||||
|
for pid in people_by_credits(db, "portrait", None):
|
||||||
|
if done + dead >= limit or down >= _PLEX_DOWN_AFTER:
|
||||||
|
break
|
||||||
|
url = cdn_url(photos.get(pid) or "")
|
||||||
|
if url is None:
|
||||||
|
continue
|
||||||
|
key = _portrait_keys(url)[0]
|
||||||
|
if imgcache.has(key) or imgcache.missed_within(key, PORTRAIT_RETRY):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
cache_portrait(plex, url)
|
||||||
|
done, down = done + 1, 0
|
||||||
|
except PlexError as exc:
|
||||||
|
# "Not found" is Plex's answer about this one file (a dead CDN headshot: measured 404,
|
||||||
|
# the CDN itself says 403). Anything else — no answer, a refused token, a 5xx — is
|
||||||
|
# Plex failing, which must neither mark the file nor keep the run going.
|
||||||
|
cause = exc.__cause__
|
||||||
|
if isinstance(cause, httpx.HTTPStatusError) and cause.response.status_code in _DEAD:
|
||||||
|
dead, down = dead + 1, 0 # Plex answered: it is up, the streak is broken
|
||||||
|
imgcache.mark_miss(key)
|
||||||
|
else:
|
||||||
|
plex_failed, down = plex_failed + 1, down + 1
|
||||||
|
log.warning(
|
||||||
|
"a portrait could not be cached",
|
||||||
|
extra=logs.event(
|
||||||
|
"plex.people.portrait_failed",
|
||||||
|
person_id=pid,
|
||||||
|
reason=type(exc.__cause__).__name__,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
failed = dead + plex_failed
|
||||||
|
return {"cached": done, **({"failed": failed} if failed else {})}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from sqlalchemy.dialects.postgresql import JSONB
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.models import PlexItem, PlexLibrary
|
from app.models import PlexItem, PlexLibrary
|
||||||
|
from app.plex.genres import hu_genre
|
||||||
|
|
||||||
TS_CONFIG = "public.unaccent_simple"
|
TS_CONFIG = "public.unaccent_simple"
|
||||||
|
|
||||||
@@ -183,7 +184,13 @@ def meta_filter_conds(model, p: dict, lang: str = "en") -> list:
|
|||||||
gsel = split_csv(p.get("genres"))
|
gsel = split_csv(p.get("genres"))
|
||||||
if gsel:
|
if gsel:
|
||||||
gcol = genres_col(model, lang)
|
gcol = genres_col(model, lang)
|
||||||
gc = [gcol.contains([g]) for g in gsel]
|
if lang == "hu":
|
||||||
|
# The HU column holds one Hungarian word per genre (plex/genres.py), so a filter saved while
|
||||||
|
# it offered "Action" is read as "Akció" too. The word as picked still matches as well: a row
|
||||||
|
# the sync has not reached since keeps its English (or Plex's mixed) list until it does.
|
||||||
|
gc = [or_(*(gcol.contains([w]) for w in dict.fromkeys((hu_genre(g), g)))) for g in gsel]
|
||||||
|
else:
|
||||||
|
gc = [gcol.contains([g]) for g in gsel]
|
||||||
conds.append(and_(*gc) if p.get("genre_mode") == "all" else or_(*gc))
|
conds.append(and_(*gc) if p.get("genre_mode") == "all" else or_(*gc))
|
||||||
crs = split_csv(p.get("content_ratings"))
|
crs = split_csv(p.get("content_ratings"))
|
||||||
if crs:
|
if crs:
|
||||||
|
|||||||
@@ -245,6 +245,9 @@ def _probe_video_kind(src: Path) -> str:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
|
# A non-zero exit is a failed probe (CalledProcessError, a SubprocessError → the branch
|
||||||
|
# below), not an answer: whatever it printed before refusing is not the file's truth.
|
||||||
|
check=True,
|
||||||
)
|
)
|
||||||
except (subprocess.SubprocessError, OSError) as exc:
|
except (subprocess.SubprocessError, OSError) as exc:
|
||||||
# The safe kind is a real answer, but it is the answer for an 8-bit SDR source — a blind
|
# The safe kind is a real answer, but it is the answer for an 8-bit SDR source — a blind
|
||||||
@@ -502,6 +505,9 @@ def _probe_audio_langs(src: Path) -> list[str]:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
|
# A non-zero exit is a failed probe (CalledProcessError, a SubprocessError → the branch
|
||||||
|
# below), not an answer: whatever it printed before refusing is not the file's truth.
|
||||||
|
check=True,
|
||||||
)
|
)
|
||||||
return [ln.strip() for ln in out.stdout.splitlines()]
|
return [ln.strip() for ln in out.stdout.splitlines()]
|
||||||
except (subprocess.SubprocessError, OSError) as exc:
|
except (subprocess.SubprocessError, OSError) as exc:
|
||||||
@@ -615,6 +621,9 @@ def _probe_first_pts(seg: Path) -> float | None:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
|
# A non-zero exit is a failed probe (CalledProcessError, a SubprocessError → the branch
|
||||||
|
# below), not an answer: whatever it printed before refusing is not the file's truth.
|
||||||
|
check=True,
|
||||||
)
|
)
|
||||||
for line in out.stdout.splitlines():
|
for line in out.stdout.splitlines():
|
||||||
tok = line.strip().strip(",")
|
tok = line.strip().strip(",")
|
||||||
@@ -736,8 +745,7 @@ def start_session(
|
|||||||
# check-and-replace happens under ONE _lock, so a newer same-key session (a concurrent
|
# check-and-replace happens under ONE _lock, so a newer same-key session (a concurrent
|
||||||
# seek-restart) installed between the fallback's decision and here is never clobbered. When the
|
# seek-restart) installed between the fallback's decision and here is never clobbered. When the
|
||||||
# registered session is no longer `replace_only`, we abort and return None (the caller yields to
|
# registered session is no longer `replace_only`, we abort and return None (the caller yields to
|
||||||
# whoever is registered now). The INITIAL spawn passes replace_only=None and always supersedes —
|
# whoever is registered now). The INITIAL spawn never gets None — see its call site.
|
||||||
# that is the seek-restart's intended last-writer-wins.
|
|
||||||
# Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and
|
# Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and
|
||||||
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
||||||
preset, crf, max_height = "veryfast", 20, 0
|
preset, crf, max_height = "veryfast", 20, 0
|
||||||
@@ -814,6 +822,7 @@ def start_session(
|
|||||||
return s
|
return s
|
||||||
|
|
||||||
s = _spawn(*_select_encode(db, item, transcode, src))
|
s = _spawn(*_select_encode(db, item, transcode, src))
|
||||||
|
assert s is not None # replace_only=None always supersedes: a seek-restart's last-writer-wins
|
||||||
# Wait for the first video segment outside the lock (this blocks on ffmpeg, and we must not hold up
|
# Wait for the first video segment outside the lock (this blocks on ffmpeg, and we must not hold up
|
||||||
# other sessions). Pass the proc so a HW session that dies at VAAPI init fails fast (not after 25s).
|
# other sessions). Pass the proc so a HW session that dies at VAAPI init fails fast (not after 25s).
|
||||||
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ window still took the full 145 s and emitted the whole track), so windows are ex
|
|||||||
zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import codecs
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
@@ -161,7 +162,8 @@ def ffmpeg_to_vtt(src: str, sub_ord: int | None) -> str:
|
|||||||
"webvtt",
|
"webvtt",
|
||||||
"pipe:1",
|
"pipe:1",
|
||||||
]
|
]
|
||||||
r = subprocess.run(cmd, capture_output=True, timeout=EXTRACT_TIMEOUT_S)
|
# check=False: the exit status is read on the next line, with ffmpeg's own stderr as the reason.
|
||||||
|
r = subprocess.run(cmd, capture_output=True, timeout=EXTRACT_TIMEOUT_S, check=False)
|
||||||
if r.returncode != 0 or not r.stdout:
|
if r.returncode != 0 or not r.stdout:
|
||||||
raise PlexError(
|
raise PlexError(
|
||||||
(r.stderr or b"").decode("utf-8", "replace")[:200] or "ffmpeg subtitle convert failed"
|
(r.stderr or b"").decode("utf-8", "replace")[:200] or "ffmpeg subtitle convert failed"
|
||||||
@@ -217,7 +219,11 @@ _FULL_END_S = 1_000_000_000.0 # stand-in for "to the end of the file" (JSON can
|
|||||||
|
|
||||||
|
|
||||||
def cache_key(rating_key: str, sub_ord: int) -> str:
|
def cache_key(rating_key: str, sub_ord: int) -> str:
|
||||||
return f"sub_{rating_key}_{sub_ord}"
|
# Versioned whenever `decode_subtitle` changes what a track decodes to — `sub2_` when legacy code
|
||||||
|
# pages were cached as U+FFFD, `sub3_` when a stray byte, an untagged file or Serbian still went
|
||||||
|
# to the wrong code page. The cache has no eviction, so every track is read again once rather
|
||||||
|
# than a broken one being served for good; the older files are just left behind.
|
||||||
|
return f"sub3_{rating_key}_{sub_ord}"
|
||||||
|
|
||||||
|
|
||||||
def _vtt_path(key: str) -> Path:
|
def _vtt_path(key: str) -> Path:
|
||||||
@@ -904,15 +910,90 @@ def _full_pass_start(rating_key: str, targets: dict[int, int]) -> float:
|
|||||||
return max(0.0, float(f"{min(ends) - WINDOW_PRE_S:.3f}"))
|
return max(0.0, float(f"{min(ends) - WINDOW_PRE_S:.3f}"))
|
||||||
|
|
||||||
|
|
||||||
|
# The legacy code page a subtitle in this language (Plex's ISO 639-2 `languageCode`) was most likely
|
||||||
|
# saved in. Serbian sits here, not with Cyrillic: its releases are overwhelmingly Latin script in
|
||||||
|
# Windows-1250, like Croatian and Bosnian.
|
||||||
|
_CP1250 = {
|
||||||
|
"hun",
|
||||||
|
"ces",
|
||||||
|
"cze",
|
||||||
|
"pol",
|
||||||
|
"slk",
|
||||||
|
"slo",
|
||||||
|
"slv",
|
||||||
|
"hrv",
|
||||||
|
"bos",
|
||||||
|
"srp",
|
||||||
|
"ron",
|
||||||
|
"rum",
|
||||||
|
"sqi",
|
||||||
|
"alb",
|
||||||
|
}
|
||||||
|
_CP1251 = {"rus", "ukr", "bel", "bul", "mkd", "mac"}
|
||||||
|
# Western European languages keep Windows-1252 (cp1250 has no à/è/ñ). An UNTAGGED file falls to
|
||||||
|
# Windows-1250: its ASCII and á/é/í/ó/ö/ú/ü match Windows-1252 byte for byte, and it also gets ő/ű
|
||||||
|
# right, which Windows-1252 turns into õ/û — the library this reads is mostly Hungarian.
|
||||||
|
_CP1252 = {
|
||||||
|
"eng",
|
||||||
|
"fre",
|
||||||
|
"fra",
|
||||||
|
"ger",
|
||||||
|
"deu",
|
||||||
|
"spa",
|
||||||
|
"ita",
|
||||||
|
"por",
|
||||||
|
"dut",
|
||||||
|
"nld",
|
||||||
|
"swe",
|
||||||
|
"dan",
|
||||||
|
"nor",
|
||||||
|
"fin",
|
||||||
|
"ice",
|
||||||
|
"isl",
|
||||||
|
"cat",
|
||||||
|
}
|
||||||
|
# A file that is almost all valid UTF-8 is UTF-8 with a few stray bytes (a pasted Windows-1252 quote),
|
||||||
|
# not a legacy file: accents in a legacy code page are nearly never valid UTF-8 sequences, so a real
|
||||||
|
# legacy file has hardly any. At least this many valid multi-byte characters per invalid byte.
|
||||||
|
_UTF8_VALID_PER_STRAY = 10
|
||||||
|
|
||||||
|
|
||||||
|
def decode_subtitle(raw: bytes, language_code: str | None) -> str:
|
||||||
|
"""A sidecar subtitle file as text. SubRip carries no charset, and a Hungarian release is
|
||||||
|
typically Windows-1250: decoded as UTF-8 with `replace`, every ő/ű/é came out as U+FFFD (found
|
||||||
|
live, 1391 of them in one episode). So: a BOM decides; else valid UTF-8 stays UTF-8 (a legacy
|
||||||
|
code page with accents is almost never valid UTF-8), and so does NEARLY valid UTF-8 — one stray
|
||||||
|
byte must not send a whole UTF-8 file to a legacy code page; else the language's legacy code page."""
|
||||||
|
if raw.startswith(codecs.BOM_UTF8):
|
||||||
|
return raw[len(codecs.BOM_UTF8) :].decode("utf-8", "replace")
|
||||||
|
if raw.startswith((codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)):
|
||||||
|
return raw.decode("utf-16", "replace")
|
||||||
|
try:
|
||||||
|
return raw.decode("utf-8")
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
# silent-ok: not strictly UTF-8 — nearly-UTF-8 or a legacy code page, told apart below.
|
||||||
|
pass
|
||||||
|
lenient = raw.decode("utf-8", "replace")
|
||||||
|
stray = lenient.count("�")
|
||||||
|
valid = sum(1 for ch in lenient if ch > "" and ch != "�")
|
||||||
|
if valid >= stray * _UTF8_VALID_PER_STRAY:
|
||||||
|
return lenient
|
||||||
|
lang = (language_code or "").lower()
|
||||||
|
legacy = "cp1251" if lang in _CP1251 else "cp1252" if lang in _CP1252 else "cp1250"
|
||||||
|
return raw.decode(legacy, "replace")
|
||||||
|
|
||||||
|
|
||||||
def _sidecar_vtt(db: Session, stream: dict) -> str:
|
def _sidecar_vtt(db: Session, stream: dict) -> str:
|
||||||
"""An external subtitle file, fetched through Plex and converted to WebVTT."""
|
"""An external subtitle file, fetched through Plex and converted to WebVTT. Its bytes are made
|
||||||
|
UTF-8 first (`decode_subtitle`), on the ffmpeg path too: ffmpeg reads a text subtitle as UTF-8."""
|
||||||
with PlexClient(db) as plex:
|
with PlexClient(db) as plex:
|
||||||
raw = plex.raw_get(stream["key"])
|
raw = plex.raw_get(stream["key"])
|
||||||
|
text = decode_subtitle(raw, stream.get("languageCode"))
|
||||||
codec = (stream.get("codec") or stream.get("format") or "").lower()
|
codec = (stream.get("codec") or stream.get("format") or "").lower()
|
||||||
if codec in ("srt", "subrip", ""):
|
if codec in ("srt", "subrip", ""):
|
||||||
return srt_to_vtt(raw.decode("utf-8", "replace"))
|
return srt_to_vtt(text)
|
||||||
with tempfile.NamedTemporaryFile(suffix="." + (codec or "sub"), delete=False) as fh:
|
with tempfile.NamedTemporaryFile(suffix="." + (codec or "sub"), delete=False) as fh:
|
||||||
fh.write(raw)
|
fh.write(text.encode("utf-8"))
|
||||||
tmp = fh.name
|
tmp = fh.name
|
||||||
try:
|
try:
|
||||||
return ffmpeg_to_vtt(tmp, None)
|
return ffmpeg_to_vtt(tmp, None)
|
||||||
|
|||||||
+63
-83
@@ -22,8 +22,9 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app import logs, sysconfig
|
from app import logs, sysconfig
|
||||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||||
from app.plex import detail, people
|
from app.plex import detail, overlay, people
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
|
from app.plex.genres import hu_genres
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
@@ -192,20 +193,6 @@ def _enabled_section_keys(db: Session) -> set[str] | None:
|
|||||||
return keys or None
|
return keys or None
|
||||||
|
|
||||||
|
|
||||||
def _overlay_map(db: Session) -> dict[str, str]:
|
|
||||||
"""The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
|
|
||||||
('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
|
|
||||||
Malformed pairs are skipped."""
|
|
||||||
raw = sysconfig.get_str(db, "plex_overlay_map")
|
|
||||||
out: dict[str, str] = {}
|
|
||||||
for pair in raw.split(","):
|
|
||||||
hu, sep, en = pair.partition(":")
|
|
||||||
hu, en = hu.strip(), en.strip()
|
|
||||||
if sep and hu and en:
|
|
||||||
out[hu] = en
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def sync(db: Session) -> dict:
|
def sync(db: Session) -> dict:
|
||||||
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
|
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
|
||||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||||
@@ -221,7 +208,7 @@ def sync(db: Session) -> dict:
|
|||||||
"pruned": 0,
|
"pruned": 0,
|
||||||
}
|
}
|
||||||
wanted = _enabled_section_keys(db)
|
wanted = _enabled_section_keys(db)
|
||||||
overlay_map = _overlay_map(db) # {huKey: enKey}
|
overlay_map = overlay.overlay_map(db) # {huKey: enKey}
|
||||||
overlay_keys = set(overlay_map)
|
overlay_keys = set(overlay_map)
|
||||||
try:
|
try:
|
||||||
with PlexClient(db) as plex:
|
with PlexClient(db) as plex:
|
||||||
@@ -265,13 +252,13 @@ def sync(db: Session) -> dict:
|
|||||||
lib.metadata_lang = "hu"
|
lib.metadata_lang = "hu"
|
||||||
lib.overlay_of = en_lib.id
|
lib.overlay_of = en_lib.id
|
||||||
db.flush()
|
db.flush()
|
||||||
_sync_overlay(db, plex, lib, en_lib.id, stats)
|
overlay.sync_overlay(db, plex, lib, en_lib.id, stats)
|
||||||
db.commit()
|
db.commit()
|
||||||
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
|
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
|
||||||
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
||||||
try:
|
try:
|
||||||
stats["people_enriched"] = _enrich_people_batch(db, plex)
|
stats["people_enriched"] = _enrich_people_batch(db, plex)
|
||||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
except Exception: # best-effort; never fail the sync over a backfill
|
||||||
# With the traceback: a blind catch over a batch that fetches AND writes can be
|
# With the traceback: a blind catch over a batch that fetches AND writes can be
|
||||||
# a Plex shape change, a DB error or our own bug, and `%s` told them apart badly.
|
# a Plex shape change, a DB error or our own bug, and `%s` told them apart badly.
|
||||||
log.exception("the Plex people-enrich batch failed")
|
log.exception("the Plex people-enrich batch failed")
|
||||||
@@ -280,7 +267,7 @@ def sync(db: Session) -> dict:
|
|||||||
# successive runs. Only HW-decodable-codec transcodes are probed — the rest are SW by codec.
|
# successive runs. Only HW-decodable-codec transcodes are probed — the rest are SW by codec.
|
||||||
try:
|
try:
|
||||||
stats["media_probed"] = _probe_media_batch(db, plex)
|
stats["media_probed"] = _probe_media_batch(db, plex)
|
||||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
except Exception: # best-effort; never fail the sync over a backfill
|
||||||
log.exception("the Plex media-probe batch failed")
|
log.exception("the Plex media-probe batch failed")
|
||||||
# silent-ok: the return value IS the report. `{"skipped": …}` goes to the scheduler, which
|
# silent-ok: the return value IS the report. `{"skipped": …}` goes to the scheduler, which
|
||||||
# logs the job result, and to the admin route, which answers it to the caller.
|
# logs the job result, and to the admin route, which answers it to the caller.
|
||||||
@@ -294,7 +281,7 @@ def sync(db: Session) -> dict:
|
|||||||
# appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata.
|
# appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata.
|
||||||
try:
|
try:
|
||||||
stats["people_pruned"] = people.prune_orphan_people(db)
|
stats["people_pruned"] = people.prune_orphan_people(db)
|
||||||
except Exception: # noqa: BLE001 — best-effort housekeeping; never fail the sync over it
|
except Exception: # best-effort housekeeping; never fail the sync over it
|
||||||
log.exception("the Plex people prune failed")
|
log.exception("the Plex people prune failed")
|
||||||
log.info("Plex sync done: %s", stats)
|
log.info("Plex sync done: %s", stats)
|
||||||
return stats
|
return stats
|
||||||
@@ -431,7 +418,10 @@ def _paginate(plex: PlexClient, key: str, item_type: int, progress: dict | None
|
|||||||
if progress is not None:
|
if progress is not None:
|
||||||
# A truthy `total` we reached = a trustworthy complete listing. `total` unknown/zero-with-rows
|
# A truthy `total` we reached = a trustworthy complete listing. `total` unknown/zero-with-rows
|
||||||
# (a Plex that doesn't report totalSize) is treated as NOT complete, so prune stays conservative.
|
# (a Plex that doesn't report totalSize) is treated as NOT complete, so prune stays conservative.
|
||||||
|
# `total` itself is kept for a caller to whom a server that SAID 0 is news it can trust (the
|
||||||
|
# watch reconcile); an empty mirror or overlay must not read it so (E32 S2).
|
||||||
progress["complete"] = bool(total) and start >= total
|
progress["complete"] = bool(total) and start >= total
|
||||||
|
progress["total"] = total
|
||||||
|
|
||||||
|
|
||||||
# The section listing exposes at most the top ~20 cast (Plex's own cap); a stored cast longer than this
|
# The section listing exposes at most the top ~20 cast (Plex's own cap); a stored cast longer than this
|
||||||
@@ -448,6 +438,10 @@ def _apply_facet_fields(row, meta: dict) -> None:
|
|||||||
row.studio = (meta.get("studio") or None) and str(meta.get("studio"))[:255]
|
row.studio = (meta.get("studio") or None) and str(meta.get("studio"))[:255]
|
||||||
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
|
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
|
||||||
row.genres = _tags(meta, "Genre")
|
row.genres = _tags(meta, "Genre")
|
||||||
|
# Every row has Hungarian genres, not only one with an HU overlay twin (the overlay pass, which runs
|
||||||
|
# after this, puts the twin's own list over it): the HU genre facets and filter read this column,
|
||||||
|
# and an English word there could never match a filter read in Hungarian.
|
||||||
|
row.genres_hu = hu_genres(row.genres)
|
||||||
row.directors = _tags(meta, "Director")
|
row.directors = _tags(meta, "Director")
|
||||||
# The listing carries only the top ~20 cast, so re-applying it would DOWNGRADE an already-enriched
|
# The listing carries only the top ~20 cast, so re-applying it would DOWNGRADE an already-enriched
|
||||||
# full cast (up to 24, grown by the rich backfill/info-page enrich), silently breaking the
|
# full cast (up to 24, grown by the rich backfill/info-page enrich), silently breaking the
|
||||||
@@ -655,59 +649,6 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
|||||||
_prune_stale(db, shows, shows_seen, stats, complete=shows_progress.get("complete", False))
|
_prune_stale(db, shows, shows_seen, stats, complete=shows_progress.get("complete", False))
|
||||||
|
|
||||||
|
|
||||||
def _sync_overlay(
|
|
||||||
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
|
||||||
) -> None:
|
|
||||||
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
|
||||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/genres differ
|
|
||||||
by language; the language-neutral people_text feeds both search vectors.
|
|
||||||
|
|
||||||
This is a full RECONCILE: every guid-bearing EN row's HU fields are cleared first, then the HU
|
|
||||||
section repopulates the matches — so an EN row whose HU twin vanished (or was re-guid'd) falls back
|
|
||||||
to EN instead of keeping stale Hungarian metadata. HU items with no EN match are skipped."""
|
|
||||||
if lib.kind == "movie":
|
|
||||||
rows = db.execute(
|
|
||||||
select(PlexItem).where(
|
|
||||||
PlexItem.library_id == en_lib_id,
|
|
||||||
PlexItem.kind == "movie",
|
|
||||||
PlexItem.guid.is_not(None),
|
|
||||||
)
|
|
||||||
).scalars()
|
|
||||||
item_type = _TYPE_MOVIE
|
|
||||||
else:
|
|
||||||
rows = db.execute(
|
|
||||||
select(PlexShow).where(PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None))
|
|
||||||
).scalars()
|
|
||||||
item_type = _TYPE_SHOW
|
|
||||||
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
|
||||||
by_guid: dict[str, list] = {}
|
|
||||||
for r in rows:
|
|
||||||
by_guid.setdefault(r.guid, []).append(r)
|
|
||||||
# Reconcile step: clear HU fields on all guid-bearing rows. Only rows that actually held HU data
|
|
||||||
# become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is cheap; matches are re-set below.
|
|
||||||
for group in by_guid.values():
|
|
||||||
for row in group:
|
|
||||||
row.title_hu = row.summary_hu = row.genres_hu = None
|
|
||||||
seen: set[str] = (
|
|
||||||
set()
|
|
||||||
) # first HU entry for a guid wins; a dup HU item can't overwrite or double-count
|
|
||||||
for meta in _paginate(plex, lib.plex_key, item_type):
|
|
||||||
g = _canonical_guid(meta)
|
|
||||||
targets = by_guid.get(g) if g else None
|
|
||||||
if not targets or g in seen:
|
|
||||||
continue
|
|
||||||
seen.add(g)
|
|
||||||
title_hu = (meta.get("title") or "").strip() or None
|
|
||||||
summary_hu = meta.get("summary")
|
|
||||||
genres_hu = _tags(meta, "Genre")
|
|
||||||
for row in targets:
|
|
||||||
row.title_hu = title_hu
|
|
||||||
row.summary_hu = summary_hu
|
|
||||||
row.genres_hu = genres_hu
|
|
||||||
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
|
||||||
db.flush()
|
|
||||||
|
|
||||||
|
|
||||||
def _upsert_collection(
|
def _upsert_collection(
|
||||||
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
|
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
|
||||||
) -> PlexCollection:
|
) -> PlexCollection:
|
||||||
@@ -733,9 +674,11 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
|||||||
meta = plex.metadata(rk) or {}
|
meta = plex.metadata(rk) or {}
|
||||||
col = _upsert_collection(db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta)
|
col = _upsert_collection(db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta)
|
||||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||||
new_members = {
|
children = _believed_children(plex, rk, meta)
|
||||||
str(c.get("ratingKey")) for c in plex.collection_children(rk) if c.get("ratingKey")
|
if children is None:
|
||||||
}
|
db.flush()
|
||||||
|
return col
|
||||||
|
new_members = {str(c.get("ratingKey")) for c in children if c.get("ratingKey")}
|
||||||
current = db.execute(select(Model).where(Model.collection_keys.contains([rk]))).scalars().all()
|
current = db.execute(select(Model).where(Model.collection_keys.contains([rk]))).scalars().all()
|
||||||
for row in current: # drop rk from rows that are no longer members
|
for row in current: # drop rk from rows that are no longer members
|
||||||
if row.rating_key not in new_members:
|
if row.rating_key not in new_members:
|
||||||
@@ -748,6 +691,22 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
|||||||
return col
|
return col
|
||||||
|
|
||||||
|
|
||||||
|
def _believed_children(plex: PlexClient, rk: str, meta: dict) -> list[dict] | None:
|
||||||
|
"""A collection's members, or None when the answer is not believed: no members from a collection
|
||||||
|
whose own listing counts some (`childCount`) is a glitch, not a collection emptied — the same
|
||||||
|
empty-answer rule as the collection listing's (E32 review). Rebuilding on it showed the
|
||||||
|
collection empty until the next sync."""
|
||||||
|
children = plex.collection_children(rk)
|
||||||
|
count = str(meta.get("childCount") or "0")
|
||||||
|
if children or not count.isdigit() or int(count) == 0:
|
||||||
|
return children
|
||||||
|
log.warning(
|
||||||
|
"Plex listed no members for a non-empty collection; keeping its membership",
|
||||||
|
extra=logs.event("plex.sync.collection_children_empty", collection=rk),
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
||||||
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
|
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
|
||||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||||
@@ -759,6 +718,19 @@ def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
|||||||
db.flush()
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _rebuild_membership(
|
||||||
|
db: Session, lib: PlexLibrary, membership: dict[str, list[str]], kept: set[str]
|
||||||
|
) -> None:
|
||||||
|
"""Rebuild collection_keys on every member row of this library (clear + set), carrying over the
|
||||||
|
keys of a collection whose empty answer was not believed (`_believed_children`)."""
|
||||||
|
for Model in (PlexItem, PlexShow):
|
||||||
|
for row in db.execute(select(Model).where(Model.library_id == lib.id)).scalars():
|
||||||
|
held = [k for k in (row.collection_keys or []) if k in kept]
|
||||||
|
keys = [*membership.get(row.rating_key, []), *held]
|
||||||
|
row.collection_keys = list(dict.fromkeys(keys)) or None
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
||||||
"""Mirror a library's collections + membership. Reads never touch Plex afterwards. Membership is
|
"""Mirror a library's collections + membership. Reads never touch Plex afterwards. Membership is
|
||||||
written as `collection_keys` on member movies (plex_items) / shows (plex_shows). A full reconcile:
|
written as `collection_keys` on member movies (plex_items) / shows (plex_shows). A full reconcile:
|
||||||
@@ -770,7 +742,17 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
|||||||
select(PlexCollection).where(PlexCollection.library_id == lib.id)
|
select(PlexCollection).where(PlexCollection.library_id == lib.id)
|
||||||
).scalars()
|
).scalars()
|
||||||
}
|
}
|
||||||
|
if not cols and existing:
|
||||||
|
# The catalog's own rule (see the module doc): an empty listing while we hold rows is a Plex
|
||||||
|
# glitch, not a library emptied of collections. Reconciling on it dropped every collection
|
||||||
|
# (its user-set `editable` flag with it) and every member's keys (E32 S2).
|
||||||
|
log.warning(
|
||||||
|
"Plex listed no collections for a library that has some; keeping them this run",
|
||||||
|
extra=logs.event("plex.sync.collections_empty", library=lib.plex_key),
|
||||||
|
)
|
||||||
|
return
|
||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
|
kept: set[str] = set() # collections whose empty children answer is not believed
|
||||||
membership: dict[str, list[str]] = {} # member rating_key -> [collection rating_key, ...]
|
membership: dict[str, list[str]] = {} # member rating_key -> [collection rating_key, ...]
|
||||||
for meta in cols:
|
for meta in cols:
|
||||||
rk = str(meta.get("ratingKey") or "")
|
rk = str(meta.get("ratingKey") or "")
|
||||||
@@ -778,7 +760,10 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
|||||||
continue
|
continue
|
||||||
_upsert_collection(db, existing.get(rk), rk, lib.id, meta)
|
_upsert_collection(db, existing.get(rk), rk, lib.id, meta)
|
||||||
seen.add(rk)
|
seen.add(rk)
|
||||||
for child in plex.collection_children(rk):
|
children = _believed_children(plex, rk, meta)
|
||||||
|
if children is None:
|
||||||
|
kept.add(rk)
|
||||||
|
for child in children or []:
|
||||||
crk = str(child.get("ratingKey") or "")
|
crk = str(child.get("ratingKey") or "")
|
||||||
if crk:
|
if crk:
|
||||||
membership.setdefault(crk, []).append(rk)
|
membership.setdefault(crk, []).append(rk)
|
||||||
@@ -787,9 +772,4 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
|||||||
for rk, col in existing.items():
|
for rk, col in existing.items():
|
||||||
if rk not in seen:
|
if rk not in seen:
|
||||||
db.delete(col)
|
db.delete(col)
|
||||||
# Rebuild collection_keys on every member row of this library (clear + set).
|
_rebuild_membership(db, lib, membership, kept)
|
||||||
for row in db.execute(select(PlexItem).where(PlexItem.library_id == lib.id)).scalars():
|
|
||||||
row.collection_keys = membership.get(row.rating_key) or None
|
|
||||||
for row in db.execute(select(PlexShow).where(PlexShow.library_id == lib.id)).scalars():
|
|
||||||
row.collection_keys = membership.get(row.rating_key) or None
|
|
||||||
db.flush()
|
|
||||||
|
|||||||
+180
-27
@@ -10,8 +10,10 @@ One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `
|
|||||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from typing import cast
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
@@ -21,6 +23,8 @@ from sqlalchemy.orm import Session
|
|||||||
from app import logs, sysconfig
|
from app import logs, sysconfig
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
from app.models import PlexItem, PlexPerson, PlexShow
|
from app.models import PlexItem, PlexPerson, PlexShow
|
||||||
|
from app.plex.cards import display_title
|
||||||
|
from app.plex.people import people_by_credits, warm_portraits
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||||
@@ -124,9 +128,17 @@ class TmdbClient:
|
|||||||
"""Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find."""
|
"""Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find."""
|
||||||
return self._get(f"/find/{external_id}", {"external_source": source})
|
return self._get(f"/find/{external_id}", {"external_source": source})
|
||||||
|
|
||||||
|
def countries(self) -> list[dict] | None:
|
||||||
|
"""TMDB's country list: `[{iso_3166_1, english_name, native_name}]` (a JSON array, which
|
||||||
|
`_get`'s dict annotation does not describe)."""
|
||||||
|
return cast("list[dict] | None", self._get("/configuration/countries"))
|
||||||
|
|
||||||
def fetch_person(self, person_id: int) -> dict | None:
|
def fetch_person(self, person_id: int) -> dict | None:
|
||||||
"""One person's bio + combined credits (E5 S6), in a single append_to_response request."""
|
"""One person's bio + combined credits (E5 S6) + the bio's translations (E30 S4), in a single
|
||||||
return self._get(f"/person/{person_id}", {"append_to_response": "combined_credits"})
|
append_to_response request."""
|
||||||
|
return self._get(
|
||||||
|
f"/person/{person_id}", {"append_to_response": "combined_credits,translations"}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None:
|
def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None:
|
||||||
@@ -254,8 +266,12 @@ def parse_tmdb(payload: dict, media_type: str) -> dict:
|
|||||||
c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")
|
c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")
|
||||||
][:8]
|
][:8]
|
||||||
or None,
|
or None,
|
||||||
|
# The ISO code beside TMDB's English name (E30): the client names the country in the reader's
|
||||||
|
# language from the code; the name is the fallback for a code it cannot name.
|
||||||
"production_countries": [
|
"production_countries": [
|
||||||
c.get("name") for c in (payload.get("production_countries") or []) if c.get("name")
|
{"code": c.get("iso_3166_1") or None, "name": c["name"]}
|
||||||
|
for c in (payload.get("production_countries") or [])
|
||||||
|
if c.get("name")
|
||||||
]
|
]
|
||||||
or None,
|
or None,
|
||||||
"recommendations": recs or None,
|
"recommendations": recs or None,
|
||||||
@@ -301,8 +317,8 @@ def _link_people(db: Session, links: dict[str, int]) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the remainder —
|
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the
|
||||||
mirrors _enrich_people_batch). A transient TmdbError leaves the watermark NULL for retry; a
|
remainder). A transient TmdbError leaves the watermark NULL for retry; a
|
||||||
definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched
|
definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched
|
||||||
every run. One commit at the end. Returns the number processed (matched OR definitively unmatched)."""
|
every run. One commit at the end. Returns the number processed (matched OR definitively unmatched)."""
|
||||||
movies = (
|
movies = (
|
||||||
@@ -356,13 +372,62 @@ def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
|||||||
return processed
|
return processed
|
||||||
|
|
||||||
|
|
||||||
|
def _country(c: dict | str) -> dict:
|
||||||
|
"""One stored production country as served: `{code, name}`. A row enriched before E30 stored the
|
||||||
|
bare English name; until `backfill_country_codes` reaches it, it serves with no code (the client
|
||||||
|
shows the name)."""
|
||||||
|
return c if isinstance(c, dict) else {"code": None, "name": c}
|
||||||
|
|
||||||
|
|
||||||
|
def backfill_country_codes(db: Session, tmdb: TmdbClient) -> int:
|
||||||
|
"""Give the countries stored before E30 (bare English names) their ISO codes, from TMDB's own
|
||||||
|
country list — the names came from that list, so they match it exactly. One request, and only
|
||||||
|
while such a row is left: re-fetching each title instead would take the enrichment batch days.
|
||||||
|
A name the list does not hold keeps no code. Returns the number of rows rewritten."""
|
||||||
|
stale = [
|
||||||
|
row
|
||||||
|
for model in (PlexItem, PlexShow)
|
||||||
|
for row in db.execute(
|
||||||
|
select(model).where(func.jsonb_typeof(model.production_countries[0]) == "string")
|
||||||
|
).scalars()
|
||||||
|
]
|
||||||
|
if not stale:
|
||||||
|
return 0
|
||||||
|
codes = {
|
||||||
|
c["english_name"]: c["iso_3166_1"]
|
||||||
|
for c in tmdb.countries() or []
|
||||||
|
if c.get("english_name") and c.get("iso_3166_1")
|
||||||
|
}
|
||||||
|
for row in stale:
|
||||||
|
row.production_countries = [
|
||||||
|
{"code": codes.get(c), "name": c} if isinstance(c, str) else c
|
||||||
|
for c in row.production_countries or []
|
||||||
|
]
|
||||||
|
db.commit()
|
||||||
|
return len(stale)
|
||||||
|
|
||||||
|
|
||||||
def run_enrich(db: Session) -> dict:
|
def run_enrich(db: Session) -> dict:
|
||||||
"""Scheduler entry point. No-op (never an error) when the TMDB key isn't configured."""
|
"""Scheduler entry point. No-op (never an error) when the TMDB key isn't configured."""
|
||||||
if not sysconfig.get_str(db, "tmdb_api_key"):
|
if not sysconfig.get_str(db, "tmdb_api_key"):
|
||||||
return {"enriched": 0, "skipped": "tmdb_api_key not set"}
|
return {"enriched": 0, "skipped": "tmdb_api_key not set"}
|
||||||
with TmdbClient(db) as tmdb:
|
with TmdbClient(db) as tmdb:
|
||||||
|
coded = backfill_country_codes(db, tmdb)
|
||||||
n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch)
|
n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch)
|
||||||
return {"enriched": n}
|
return {"enriched": n, **({"countries_coded": coded} if coded else {})}
|
||||||
|
|
||||||
|
|
||||||
|
def run_people_enrich(db: Session) -> dict:
|
||||||
|
"""Scheduler entry point of people_enrich (E30 S6): a batch of person bios (the most-credited
|
||||||
|
first) and a batch of portraits. Each half is a no-op — never an error — without its TMDB key /
|
||||||
|
without Plex."""
|
||||||
|
batch = settings.people_enrich_batch
|
||||||
|
if sysconfig.get_str(db, "tmdb_api_key"):
|
||||||
|
with TmdbClient(db) as tmdb:
|
||||||
|
bios = enrich_people_batch(db, tmdb, batch)
|
||||||
|
else:
|
||||||
|
bios = {"skipped": "tmdb_api_key not set"}
|
||||||
|
return {"bios": bios, "portraits": warm_portraits(db, batch)}
|
||||||
|
|
||||||
|
|
||||||
def resolve_library_guids(
|
def resolve_library_guids(
|
||||||
@@ -406,10 +471,11 @@ def resolve_library_guids(
|
|||||||
return movies, shows
|
return movies, shows
|
||||||
|
|
||||||
|
|
||||||
def resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]:
|
def resolve_recs(db: Session, recs: list[dict] | None, *, lang: str) -> list[dict]:
|
||||||
"""Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:<id>`, bucketed
|
"""Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:<id>`, bucketed
|
||||||
by media_type — see resolve_library_guids) and, if so, its rating_key — so the client can deep-link
|
by media_type — see resolve_library_guids) and, if so, its rating_key — so the client can deep-link
|
||||||
the ones we have and merely display the rest."""
|
the ones we have and merely display the rest. One we have is named as the library names it, in
|
||||||
|
`lang` (E30); one we lack keeps the title TMDB gave."""
|
||||||
if not recs:
|
if not recs:
|
||||||
return []
|
return []
|
||||||
movies, shows = resolve_library_guids(db, recs)
|
movies, shows = resolve_library_guids(db, recs)
|
||||||
@@ -417,13 +483,19 @@ def resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]:
|
|||||||
for r in recs:
|
for r in recs:
|
||||||
g = f"tmdb:{r.get('tmdb_id')}"
|
g = f"tmdb:{r.get('tmdb_id')}"
|
||||||
row = shows.get(g) if r.get("media_type") == "tv" else movies.get(g)
|
row = shows.get(g) if r.get("media_type") == "tv" else movies.get(g)
|
||||||
|
named = {"title": display_title(row, lang)} if row is not None else {}
|
||||||
out.append(
|
out.append(
|
||||||
{**r, "in_library": row is not None, "rating_key": row.rating_key if row else None}
|
{
|
||||||
|
**r,
|
||||||
|
**named,
|
||||||
|
"in_library": row is not None,
|
||||||
|
"rating_key": row.rating_key if row else None,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def serve_block(db: Session, row) -> dict | None:
|
def serve_block(db: Session, row, *, lang: str) -> dict | None:
|
||||||
"""The TMDB extras for a movie/show detail response, served from the row (no live TMDB call).
|
"""The TMDB extras for a movie/show detail response, served from the row (no live TMDB call).
|
||||||
None until the row has been TMDB-enriched (the client then shows nothing extra)."""
|
None until the row has been TMDB-enriched (the client then shows nothing extra)."""
|
||||||
if row.tmdb_enriched_at is None:
|
if row.tmdb_enriched_at is None:
|
||||||
@@ -437,30 +509,58 @@ def serve_block(db: Session, row) -> dict | None:
|
|||||||
"keywords": row.keywords or [],
|
"keywords": row.keywords or [],
|
||||||
"crew": row.crew or [],
|
"crew": row.crew or [],
|
||||||
"studios": row.studios or [],
|
"studios": row.studios or [],
|
||||||
"production_countries": row.production_countries or [],
|
"production_countries": [_country(c) for c in row.production_countries or []],
|
||||||
"recommendations": resolve_recs(db, row.recommendations),
|
"recommendations": resolve_recs(db, row.recommendations, lang=lang),
|
||||||
"backdrops": row.backdrops or [],
|
"backdrops": row.backdrops or [],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
# --- Person enrichment (E5 S6) -------------------------------------------------------------------
|
# --- Person enrichment (E5 S6) -------------------------------------------------------------------
|
||||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once (the first person-page open)
|
# A person's bio + birth facts + a "Known for" credit pool, fetched once — by the people_enrich job, or
|
||||||
# and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
# by the first person-page open if the job has not reached them yet — and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||||
# endpoint resolves them to library rows by guid at serve time (like recommendations).
|
# endpoint resolves them to library rows by guid at serve time (like recommendations).
|
||||||
|
|
||||||
_KNOWN_FOR_MAX = 15
|
# The candidate pool kept per person. Only the titles that are in the library are shown (at most
|
||||||
|
# routes.plex.people._KNOWN_FOR_SHOWN), so the pool is deeper than the rail: a star's 15 most-voted
|
||||||
|
# credits are often not the 15 a given library holds.
|
||||||
|
_KNOWN_FOR_MAX = 40
|
||||||
|
# TMDB genre ids of the shows a person visits as a guest (News, Reality, Talk). Ranked by popularity,
|
||||||
|
# talk-show visits crowded out the films (Tom Cruise: 14 of 15 were talk shows; E30 S6).
|
||||||
|
_GUEST_GENRES = {10763, 10764, 10767}
|
||||||
|
# A credit in which the person plays themselves ("Self", "Himself - Guest", "Herself (archive footage)").
|
||||||
|
_SELF_CREDIT = re.compile(r"^\s*(self|himself|herself|themselves)\b", re.IGNORECASE)
|
||||||
|
# The crew jobs that make a title the person's own work (a director is known for what they directed).
|
||||||
|
_KNOWN_FOR_CREW_JOBS = {"Director", "Creator"}
|
||||||
|
|
||||||
|
|
||||||
|
def _known_for_credit(c: dict, crew: bool) -> bool:
|
||||||
|
if crew:
|
||||||
|
return c.get("job") in _KNOWN_FOR_CREW_JOBS
|
||||||
|
if _SELF_CREDIT.match(c.get("character") or ""):
|
||||||
|
return False
|
||||||
|
return not _GUEST_GENRES.intersection(c.get("genre_ids") or [])
|
||||||
|
|
||||||
|
|
||||||
def parse_person(payload: dict) -> dict:
|
def parse_person(payload: dict) -> dict:
|
||||||
"""TMDB /person/{id} (+combined_credits) → the persisted bio fields. `known_for` = the person's
|
"""TMDB /person/{id} (+combined_credits) → the persisted bio fields. `known_for` = the person's
|
||||||
most-popular movie/tv credits (deduped by id), top `_KNOWN_FOR_MAX`, resolved to library later."""
|
most-voted movie/tv credits — roles and directed works, not guest visits as themselves — deduped,
|
||||||
credits = (payload.get("combined_credits") or {}).get("cast") or []
|
the top `_KNOWN_FOR_MAX`, resolved to library rows at serve time. Ranked by vote count, as TMDB's
|
||||||
|
own "Known for" is: popularity is this week's attention, which a talk-show visit wins."""
|
||||||
|
combined = payload.get("combined_credits") or {}
|
||||||
|
credits = [c for c in combined.get("cast") or [] if _known_for_credit(c, crew=False)] + [
|
||||||
|
c for c in combined.get("crew") or [] if _known_for_credit(c, crew=True)
|
||||||
|
]
|
||||||
# Dedup on (id, media_type), not the bare id: TMDB movie ids and tv ids are independent sequences
|
# Dedup on (id, media_type), not the bare id: TMDB movie ids and tv ids are independent sequences
|
||||||
# (id 603 can be both a movie and a show — see resolve_library_guids), so a same-id movie+show pair
|
# (id 603 can be both a movie and a show — see resolve_library_guids), so a same-id movie+show pair
|
||||||
# is two distinct credits and both must survive.
|
# is two distinct credits and both must survive.
|
||||||
seen: set[tuple[int, str]] = set()
|
seen: set[tuple[int, str]] = set()
|
||||||
known: list[dict] = []
|
known: list[dict] = []
|
||||||
for c in sorted(credits, key=lambda c: c.get("popularity") or 0, reverse=True):
|
ranked = sorted(
|
||||||
|
credits,
|
||||||
|
key=lambda c: (c.get("vote_count") or 0, c.get("popularity") or 0),
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
for c in ranked:
|
||||||
tid, mt = c.get("id"), c.get("media_type")
|
tid, mt = c.get("id"), c.get("media_type")
|
||||||
if not tid or mt not in ("movie", "tv") or (tid, mt) in seen:
|
if not tid or mt not in ("movie", "tv") or (tid, mt) in seen:
|
||||||
continue
|
continue
|
||||||
@@ -470,6 +570,7 @@ def parse_person(payload: dict) -> dict:
|
|||||||
break
|
break
|
||||||
return {
|
return {
|
||||||
"biography": (payload.get("biography") or "").strip() or None,
|
"biography": (payload.get("biography") or "").strip() or None,
|
||||||
|
"biography_hu": _translated_bio(payload, "hu"),
|
||||||
"birthday": payload.get("birthday") or None,
|
"birthday": payload.get("birthday") or None,
|
||||||
"deathday": payload.get("deathday") or None,
|
"deathday": payload.get("deathday") or None,
|
||||||
"place_of_birth": payload.get("place_of_birth") or None,
|
"place_of_birth": payload.get("place_of_birth") or None,
|
||||||
@@ -479,8 +580,17 @@ def parse_person(payload: dict) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _translated_bio(payload: dict, lang: str) -> str | None:
|
||||||
|
"""The `lang` biography from the appended `translations` (TMDB leaves an untranslated one blank)."""
|
||||||
|
for tr in (payload.get("translations") or {}).get("translations") or []:
|
||||||
|
if tr.get("iso_639_1") == lang:
|
||||||
|
return ((tr.get("data") or {}).get("biography") or "").strip() or None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def apply_person(row: PlexPerson, parsed: dict) -> None:
|
def apply_person(row: PlexPerson, parsed: dict) -> None:
|
||||||
row.biography = parsed["biography"]
|
row.biography = parsed["biography"]
|
||||||
|
row.biography_hu = parsed["biography_hu"]
|
||||||
row.birthday = parsed["birthday"]
|
row.birthday = parsed["birthday"]
|
||||||
row.deathday = parsed["deathday"]
|
row.deathday = parsed["deathday"]
|
||||||
row.place_of_birth = parsed["place_of_birth"]
|
row.place_of_birth = parsed["place_of_birth"]
|
||||||
@@ -490,11 +600,22 @@ def apply_person(row: PlexPerson, parsed: dict) -> None:
|
|||||||
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_person_into(tmdb: TmdbClient, person: PlexPerson) -> None:
|
||||||
|
"""Fetch one person from TMDB and write the row (not committed). A 404 (deleted person) stamps the
|
||||||
|
watermark with empty fields so it isn't fetched again. Raises TmdbError on a transient failure,
|
||||||
|
leaving the row untouched and `tmdb_enriched_at` NULL for a retry."""
|
||||||
|
payload = tmdb.fetch_person(cast(int, person.tmdb_person_id))
|
||||||
|
if payload is None:
|
||||||
|
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||||
|
else:
|
||||||
|
apply_person(person, parse_person(payload))
|
||||||
|
|
||||||
|
|
||||||
def enrich_person(db: Session, person: PlexPerson) -> bool:
|
def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||||
"""Lazily fetch + persist one person's TMDB bio (first person-page open), committing the write so it
|
"""Lazily fetch + persist one person's TMDB bio (a person page opened before the background job
|
||||||
survives regardless of the read path. No-op (False) without a key or a `tmdb_person_id`; a transient
|
reached them), committing the write so it survives regardless of the read path. No-op (False)
|
||||||
error leaves `tmdb_enriched_at` NULL so the next open retries. A 404 (deleted person) stamps the
|
without a key or a `tmdb_person_id`; a transient error leaves `tmdb_enriched_at` NULL so the next
|
||||||
watermark with empty fields so we don't refetch it every open."""
|
open retries."""
|
||||||
if not person.tmdb_person_id or not sysconfig.get_str(db, "tmdb_api_key"):
|
if not person.tmdb_person_id or not sysconfig.get_str(db, "tmdb_api_key"):
|
||||||
return False
|
return False
|
||||||
try:
|
try:
|
||||||
@@ -502,7 +623,7 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
|||||||
# longer blocks a serve; still cap the outbound TMDB call with a short timeout so a hanging TMDB
|
# longer blocks a serve; still cap the outbound TMDB call with a short timeout so a hanging TMDB
|
||||||
# doesn't tie up a threadpool worker for the default 30s.
|
# doesn't tie up a threadpool worker for the default 30s.
|
||||||
with TmdbClient(db, timeout=8.0) as tmdb:
|
with TmdbClient(db, timeout=8.0) as tmdb:
|
||||||
payload = tmdb.fetch_person(person.tmdb_person_id)
|
_fetch_person_into(tmdb, person)
|
||||||
except TmdbError as exc:
|
except TmdbError as exc:
|
||||||
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
||||||
# front of the user right now shows an empty biography with nothing to say why.
|
# front of the user right now shows an empty biography with nothing to say why.
|
||||||
@@ -515,9 +636,41 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
return False
|
return False
|
||||||
if payload is None:
|
|
||||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
|
||||||
else:
|
|
||||||
apply_person(person, parse_person(payload))
|
|
||||||
db.commit()
|
db.commit()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
# Lookups failing this many people in a row end a people batch: TMDB itself is degraded, and hammering
|
||||||
|
# it helps nobody. One person failing alone is skipped, so an id TMDB keeps refusing never stalls the rest.
|
||||||
|
_PEOPLE_FAILS_IN_A_ROW = 3
|
||||||
|
|
||||||
|
|
||||||
|
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
||||||
|
"""Fetch up to `limit` linked, not-yet-enriched people (E30 S6), the most-credited first, so a
|
||||||
|
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
||||||
|
lands: a page open during the run already sees it. A failed lookup is skipped (left unstamped for the
|
||||||
|
next run); _PEOPLE_FAILS_IN_A_ROW of them in a row end the batch."""
|
||||||
|
ids = people_by_credits(db, "bio", limit)
|
||||||
|
done = failed = in_a_row = 0
|
||||||
|
for pid in ids:
|
||||||
|
if in_a_row >= _PEOPLE_FAILS_IN_A_ROW:
|
||||||
|
break
|
||||||
|
person = db.get(PlexPerson, pid)
|
||||||
|
# A page open may have enriched this person since the select: skip, don't fetch twice.
|
||||||
|
if person is None or person.tmdb_enriched_at is not None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
_fetch_person_into(tmdb, person)
|
||||||
|
except TmdbError as exc:
|
||||||
|
db.rollback()
|
||||||
|
failed, in_a_row = failed + 1, in_a_row + 1
|
||||||
|
log.warning(
|
||||||
|
"a TMDB person lookup in the batch failed",
|
||||||
|
extra=logs.event(
|
||||||
|
"plex.tmdb.people_batch_failed", person_id=pid, reason=type(exc).__name__
|
||||||
|
),
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
db.commit()
|
||||||
|
done, in_a_row = done + 1, 0
|
||||||
|
return {"enriched": done, **({"failed": failed} if failed else {})}
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app import logs, sysconfig
|
from app import logs, sysconfig
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.models import PlexItem, PlexLink, PlexState
|
from app.models import PlexItem, PlexLibrary, PlexLink, PlexState
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
from app.plex.sync import (
|
from app.plex.sync import (
|
||||||
_TYPE_EPISODE,
|
_TYPE_EPISODE,
|
||||||
@@ -72,13 +72,27 @@ def _rk_to_id(db: Session) -> dict[str, int]:
|
|||||||
|
|
||||||
def _scan_plex_states(
|
def _scan_plex_states(
|
||||||
db: Session, plex: PlexClient
|
db: Session, plex: PlexClient
|
||||||
) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int]:
|
) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int, set[str], set[str]]:
|
||||||
"""Scan the enabled movie/show sections and return ``{item_id: (status, pos, watched_at,
|
"""Scan the enabled movie/show sections and return ``{item_id: (status, pos, watched_at,
|
||||||
prog_at)}`` for every mirrored leaf Plex holds a watch signal for, plus the scanned-leaf count.
|
prog_at)}`` for every mirrored leaf Plex holds a watch signal for, the scanned-leaf count, the
|
||||||
Shared by the one-time import and the full reconcile (both need Plex's whole current picture)."""
|
keys of the sections SETTLED — walked to the server's total (or said empty by a server we mirror
|
||||||
|
nothing from) — and the keys of every section walked.
|
||||||
|
Shared by the one-time import and the full reconcile (both need Plex's whole current picture).
|
||||||
|
The settled keys matter to the reconcile alone: it reads "absent from the scan" as "Plex
|
||||||
|
un-watched it", which a walk cut short by a Plex hiccup does not mean (see `_paginate`). Settled
|
||||||
|
is decided PER SECTION: one section that never settles (a library emptied on Plex while its
|
||||||
|
mirrored items are kept) must not stop un-watches syncing in all the others (E32 review)."""
|
||||||
item_id_by_rk = _rk_to_id(db)
|
item_id_by_rk = _rk_to_id(db)
|
||||||
wanted = _enabled_section_keys(db)
|
wanted = _enabled_section_keys(db)
|
||||||
|
# The sections we mirror something from: a total of 0 from one of those contradicts the mirror.
|
||||||
|
mirrored = set(
|
||||||
|
db.execute(
|
||||||
|
select(PlexLibrary.plex_key).join(PlexItem, PlexItem.library_id == PlexLibrary.id)
|
||||||
|
).scalars()
|
||||||
|
)
|
||||||
scanned = 0
|
scanned = 0
|
||||||
|
settled: set[str] = set()
|
||||||
|
walked: set[str] = set()
|
||||||
out: dict[int, tuple[str, int, datetime | None, datetime | None]] = {}
|
out: dict[int, tuple[str, int, datetime | None, datetime | None]] = {}
|
||||||
for s in plex.sections():
|
for s in plex.sections():
|
||||||
stype = s.get("type")
|
stype = s.get("type")
|
||||||
@@ -88,7 +102,10 @@ def _scan_plex_states(
|
|||||||
if wanted is not None and key not in wanted:
|
if wanted is not None and key not in wanted:
|
||||||
continue
|
continue
|
||||||
item_type = _TYPE_MOVIE if stype == "movie" else _TYPE_EPISODE
|
item_type = _TYPE_MOVIE if stype == "movie" else _TYPE_EPISODE
|
||||||
for meta in _paginate(plex, key, item_type):
|
walked.add(key)
|
||||||
|
progress: dict = {}
|
||||||
|
before = scanned
|
||||||
|
for meta in _paginate(plex, key, item_type, progress):
|
||||||
scanned += 1
|
scanned += 1
|
||||||
item_id = item_id_by_rk.get(str(meta.get("ratingKey") or ""))
|
item_id = item_id_by_rk.get(str(meta.get("ratingKey") or ""))
|
||||||
if item_id is None:
|
if item_id is None:
|
||||||
@@ -98,7 +115,15 @@ def _scan_plex_states(
|
|||||||
)
|
)
|
||||||
if res is not None:
|
if res is not None:
|
||||||
out[item_id] = res
|
out[item_id] = res
|
||||||
return out, scanned
|
# A section the server SAYS is empty (total 0, nothing listed) settles too. Believed only for
|
||||||
|
# a section we mirror nothing from: one whose items we hold answering 0 is a Plex that lost
|
||||||
|
# the library for a moment, not one that un-watched all of it (E32 review) — so it is left
|
||||||
|
# unsettled, and its rows are not cleared.
|
||||||
|
listed = scanned > before
|
||||||
|
said_empty = progress.get("total") == 0 and not listed and key not in mirrored
|
||||||
|
if (progress.get("complete") and listed) or said_empty:
|
||||||
|
settled.add(key)
|
||||||
|
return out, scanned, settled, walked
|
||||||
|
|
||||||
|
|
||||||
def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
||||||
@@ -111,7 +136,8 @@ def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
|||||||
stats = {"watched": 0, "in_progress": 0, "scanned": 0}
|
stats = {"watched": 0, "in_progress": 0, "scanned": 0}
|
||||||
try:
|
try:
|
||||||
with PlexClient(db) as plex:
|
with PlexClient(db) as plex:
|
||||||
scanned_states, stats["scanned"] = _scan_plex_states(db, plex)
|
# A partial scan is still safe here: the import only ever ADDS what Plex reports.
|
||||||
|
scanned_states, stats["scanned"], _settled, _walked = _scan_plex_states(db, plex)
|
||||||
# One row per item (keyed by item_id → inherently de-duped) then bulk-UPSERT. An upsert is
|
# One row per item (keyed by item_id → inherently de-duped) then bulk-UPSERT. An upsert is
|
||||||
# idempotent AND concurrency-safe: a repeated/overlapping enable (the toggle firing twice
|
# idempotent AND concurrency-safe: a repeated/overlapping enable (the toggle firing twice
|
||||||
# while the multi-second import runs) can't raise a duplicate-key error — it re-updates.
|
# while the multi-second import runs) can't raise a duplicate-key error — it re-updates.
|
||||||
@@ -590,6 +616,28 @@ def run_plex_watch_sync(db: Session) -> dict:
|
|||||||
return total
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def _clear_unwatched(db: Session, user_id: int, plex_states: dict, settled: set[str]) -> int:
|
||||||
|
"""Clear the user's rows Plex no longer has a record for — the mirrored ones (Plex un-watched
|
||||||
|
them); hidden and dirty rows stay (a dirty one is re-pushed by `_repush_dirty`). Only rows of the
|
||||||
|
sections the scan SETTLED: a walk a Plex hiccup cut short leaves out rows it merely failed to
|
||||||
|
reach, and clearing those un-watched them here until the next complete run (E32 S2); an item of
|
||||||
|
a library the allowlist leaves out is absent from every scan, and was cleared on every run (E32
|
||||||
|
review)."""
|
||||||
|
n = 0
|
||||||
|
rows = db.execute(
|
||||||
|
select(PlexState)
|
||||||
|
.join(PlexItem, PlexItem.id == PlexState.item_id)
|
||||||
|
.join(PlexLibrary, PlexLibrary.id == PlexItem.library_id)
|
||||||
|
.where(PlexState.user_id == user_id, PlexLibrary.plex_key.in_(settled))
|
||||||
|
)
|
||||||
|
for st in rows.scalars().all():
|
||||||
|
if st.item_id in plex_states or st.status == "hidden" or not st.synced_to_plex:
|
||||||
|
continue
|
||||||
|
db.delete(st)
|
||||||
|
n += 1
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
def run_plex_watch_reconcile(db: Session) -> dict:
|
def run_plex_watch_reconcile(db: Session) -> dict:
|
||||||
"""Scheduler job (default daily) — the authoritative full reconcile. Rescans every section and,
|
"""Scheduler job (default daily) — the authoritative full reconcile. Rescans every section and,
|
||||||
per item, uses `synced_to_plex` to settle what the incremental feed can't:
|
per item, uses `synced_to_plex` to settle what the incremental feed can't:
|
||||||
@@ -608,7 +656,7 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
|||||||
try:
|
try:
|
||||||
with PlexClient(db) as plex:
|
with PlexClient(db) as plex:
|
||||||
_resolve_account_id(db, plex, link)
|
_resolve_account_id(db, plex, link)
|
||||||
plex_states, scanned = _scan_plex_states(db, plex)
|
plex_states, scanned, settled, walked = _scan_plex_states(db, plex)
|
||||||
total["scanned"] += scanned
|
total["scanned"] += scanned
|
||||||
|
|
||||||
for item_id, (status, pos, watched_at, prog_at) in plex_states.items():
|
for item_id, (status, pos, watched_at, prog_at) in plex_states.items():
|
||||||
@@ -616,18 +664,19 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
|||||||
if ch:
|
if ch:
|
||||||
total["applied"] += 1
|
total["applied"] += 1
|
||||||
|
|
||||||
# Siftlode rows Plex no longer has a record for: clear the ones that were mirrored
|
total["cleared"] += _clear_unwatched(db, link.user_id, plex_states, settled)
|
||||||
# (Plex un-watched them); leave hidden + dirty (dirty gets re-pushed just below).
|
if unsettled := sorted(walked - settled):
|
||||||
for st in (
|
total["incomplete"] = total.get("incomplete", 0) + 1
|
||||||
db.execute(select(PlexState).where(PlexState.user_id == link.user_id))
|
log.warning(
|
||||||
.scalars()
|
"Plex sections %s were cut short or answered empty; their rows are not cleared"
|
||||||
.all()
|
" this run",
|
||||||
):
|
", ".join(unsettled),
|
||||||
if st.item_id in plex_states or st.status == "hidden":
|
extra=logs.event(
|
||||||
continue
|
"plex.watch.reconcile_incomplete",
|
||||||
if st.synced_to_plex:
|
user_id=link.user_id,
|
||||||
db.delete(st)
|
sections=unsettled,
|
||||||
total["cleared"] += 1
|
),
|
||||||
|
)
|
||||||
|
|
||||||
total["repushed"] += _repush_dirty(db, plex, link)
|
total["repushed"] += _repush_dirty(db, plex, link)
|
||||||
link.last_watch_sync_at = datetime.now(timezone.utc)
|
link.last_watch_sync_at = datetime.now(timezone.utc)
|
||||||
|
|||||||
+17
-2
@@ -8,7 +8,7 @@ its own thread/context) never cross wires. When no sink is bound (e.g. a manual
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import contextvars
|
import contextvars
|
||||||
from typing import Callable, Optional
|
from typing import Callable, Literal, Optional
|
||||||
|
|
||||||
Sink = Callable[[int, Optional[int], Optional[str]], None]
|
Sink = Callable[[int, Optional[int], Optional[str]], None]
|
||||||
|
|
||||||
@@ -24,6 +24,19 @@ def report(current: int, total: int | None = None, phase: str | None = None) ->
|
|||||||
sink(current, total, phase)
|
sink(current, total, phase)
|
||||||
|
|
||||||
|
|
||||||
|
class JobFailed(RuntimeError):
|
||||||
|
"""A job's own failure, said so the reader's language can name it (E31): the client reads
|
||||||
|
`scheduler.failure.<reason>` and interpolates `params` (keep them to values the reader may see —
|
||||||
|
numbers, ids, short codes). The message is the English line the log and `last_error` keep. An
|
||||||
|
upstream's own error (a timeout, a refused connection) is not one of these: it stays as raised,
|
||||||
|
being diagnostic."""
|
||||||
|
|
||||||
|
def __init__(self, reason: str, message: str, **params: object):
|
||||||
|
super().__init__(message)
|
||||||
|
self.reason = reason
|
||||||
|
self.params = params
|
||||||
|
|
||||||
|
|
||||||
class bind:
|
class bind:
|
||||||
"""Context manager that binds a progress sink for the duration of a job run."""
|
"""Context manager that binds a progress sink for the duration of a job run."""
|
||||||
|
|
||||||
@@ -35,7 +48,9 @@ class bind:
|
|||||||
self._token = _sink.set(self._sink)
|
self._token = _sink.set(self._sink)
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def __exit__(self, *exc) -> bool:
|
# Literal[False], not bool: this never swallows the job's exception, and saying so is what lets
|
||||||
|
# a type checker know a name assigned inside `with bind(...)` is bound after it.
|
||||||
|
def __exit__(self, *exc) -> Literal[False]:
|
||||||
if self._token is not None:
|
if self._token is not None:
|
||||||
_sink.reset(self._token)
|
_sink.reset(self._token)
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -28,10 +28,13 @@ def not_found(what: str = "generic") -> HTTPException:
|
|||||||
return HTTPException(status_code=404, detail={"code": "not_found", "reason": what})
|
return HTTPException(status_code=404, detail={"code": "not_found", "reason": what})
|
||||||
|
|
||||||
|
|
||||||
def get_or_404(db: Session, model: type[M], id_: Any, *, what: str = "generic") -> M:
|
def get_or_404(
|
||||||
|
db: Session, model: type[M], id_: Any, *, what: str = "generic", lock: bool = False
|
||||||
|
) -> M:
|
||||||
"""Fetch a row by primary key or raise 404. Replaces the `db.get(Model, id); if None: 404`
|
"""Fetch a row by primary key or raise 404. Replaces the `db.get(Model, id); if None: 404`
|
||||||
idiom repeated across routes."""
|
idiom repeated across routes. `lock` takes it FOR UPDATE (bypassing the identity map), so two
|
||||||
obj = db.get(model, id_)
|
requests that read-then-write its children (a playlist's item positions) run one after the other."""
|
||||||
|
obj = db.get(model, id_, with_for_update=True if lock else None)
|
||||||
if obj is None:
|
if obj is None:
|
||||||
raise not_found(what)
|
raise not_found(what)
|
||||||
return obj
|
return obj
|
||||||
@@ -45,25 +48,29 @@ def owned_or_404(
|
|||||||
*,
|
*,
|
||||||
owner_attr: str = "user_id",
|
owner_attr: str = "user_id",
|
||||||
what: str = "generic",
|
what: str = "generic",
|
||||||
|
lock: bool = False,
|
||||||
) -> M:
|
) -> M:
|
||||||
"""Fetch a row by primary key that `user` owns, or raise 404. A row owned by someone
|
"""Fetch a row by primary key that `user` owns, or raise 404 (`lock`: see `get_or_404`). A row owned by someone
|
||||||
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
||||||
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
|
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
|
||||||
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
|
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
|
||||||
obj = get_or_404(db, model, id_, what=what)
|
obj = get_or_404(db, model, id_, what=what, lock=lock)
|
||||||
if getattr(obj, owner_attr) != user.id:
|
if getattr(obj, owner_attr) != user.id:
|
||||||
raise not_found(what)
|
raise not_found(what)
|
||||||
return obj
|
return obj
|
||||||
|
|
||||||
|
|
||||||
def youtube_refusal(exc: Exception) -> HTTPException:
|
def youtube_refusal(exc: Exception, *, playlist: bool = False) -> HTTPException:
|
||||||
"""The answer to an action YouTube refused: a 422 whose reason the UI translates
|
"""The answer to an action YouTube refused: a 422 whose reason the UI translates
|
||||||
(`errors.youtube.<reason>`). One definition, because the routes had grown three shapes of it —
|
(`errors.youtube.<reason>`). One definition, because the routes had grown three shapes of it —
|
||||||
a bare English string quoting the upstream message, a playlist-scoped reason, and nothing at
|
a bare English string quoting the upstream message, a playlist-scoped reason, and nothing at
|
||||||
all (a raw 500 from the manual syncs). The upstream failure is already logged where it
|
all (a raw 500 from the manual syncs). The upstream failure is already logged where it
|
||||||
happened (`youtube.api.error`). A failure that is not the Data API's (the search scraper's) is
|
happened (`youtube.api.error`). A failure that is not the Data API's (the search scraper's) is
|
||||||
YouTube being unavailable to us — the retryable answer."""
|
YouTube being unavailable to us — the retryable answer. `playlist`: the route acts on one
|
||||||
reason = refusal_reason(exc) if isinstance(exc, YouTubeError) else "unavailable"
|
playlist the user picked, so a missing playlist is THAT one, gone (`refusal_reason`)."""
|
||||||
|
reason = (
|
||||||
|
refusal_reason(exc, playlist=playlist) if isinstance(exc, YouTubeError) else "unavailable"
|
||||||
|
)
|
||||||
return HTTPException(status_code=422, detail={"code": "youtube", "reason": reason})
|
return HTTPException(status_code=422, detail={"code": "youtube", "reason": reason})
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -834,7 +834,11 @@ def reset_demo(admin: User = Depends(admin_user), db: Session = Depends(get_db))
|
|||||||
demo = get_or_create_demo_user(db)
|
demo = get_or_create_demo_user(db)
|
||||||
seeded = reset_demo_state(db, demo)
|
seeded = reset_demo_state(db, demo)
|
||||||
audit.record(
|
audit.record(
|
||||||
db, admin.id, AuditAction.DEMO_RESET, summary=f"{seeded} sample playlists re-seeded"
|
db,
|
||||||
|
admin.id,
|
||||||
|
AuditAction.DEMO_RESET,
|
||||||
|
summary=f"{seeded} sample playlists re-seeded",
|
||||||
|
after={"playlists_seeded": seeded}, # the count as data: named in the reader's language
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
||||||
|
|||||||
@@ -66,6 +66,12 @@ def list_audit(
|
|||||||
def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||||
"""Clear the audit log. Leaves ONE row recording who cleared it + how many (tamper-evidence)."""
|
"""Clear the audit log. Leaves ONE row recording who cleared it + how many (tamper-evidence)."""
|
||||||
n = db.execute(delete(AuditLog)).rowcount or 0
|
n = db.execute(delete(AuditLog)).rowcount or 0
|
||||||
audit.record(db, admin.id, AuditAction.AUDIT_CLEAR, summary=f"Cleared {n} audit entries")
|
audit.record(
|
||||||
|
db,
|
||||||
|
admin.id,
|
||||||
|
AuditAction.AUDIT_CLEAR,
|
||||||
|
summary=f"Cleared {n} audit entries",
|
||||||
|
after={"cleared": n}, # the count as data: the log names it in the reader's language
|
||||||
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"ok": True, "cleared": n}
|
return {"ok": True, "cleared": n}
|
||||||
|
|||||||
@@ -6,9 +6,11 @@ from datetime import datetime, timezone
|
|||||||
|
|
||||||
from fastapi import APIRouter, BackgroundTasks, Depends
|
from fastapi import APIRouter, BackgroundTasks, Depends
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import Select, func, select, update
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.collation import collated, sorted_words
|
||||||
|
from app.mailtext import lang_for_user
|
||||||
from app import logs, quota
|
from app import logs, quota
|
||||||
from app.auth import admin_user, current_user, require_human, require_write_scope
|
from app.auth import admin_user, current_user, require_human, require_write_scope
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
@@ -28,7 +30,7 @@ from app.models import (
|
|||||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||||
from app.sync.explore import explore_ingest_page
|
from app.sync.explore import explore_ingest_page
|
||||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||||
from app.sync.subscriptions import apply_channel_details
|
from app.sync.subscriptions import apply_channel_details, details_stale
|
||||||
from app.utils import iso
|
from app.utils import iso
|
||||||
from app.youtube.client import YouTubeClient, YouTubeError
|
from app.youtube.client import YouTubeClient, YouTubeError
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
@@ -38,33 +40,40 @@ router = APIRouter(prefix="/api/channels", tags=["channels"])
|
|||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def channel_aggregates(channel_ids: list[str]) -> Select:
|
||||||
|
"""Per-channel stored count, last upload and total duration, in one grouped pass. Every column it
|
||||||
|
reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index
|
||||||
|
alone; a test holds that (test_channel_list_index_only)."""
|
||||||
|
return (
|
||||||
|
select(
|
||||||
|
Video.channel_id,
|
||||||
|
func.count(Video.id),
|
||||||
|
func.max(Video.published_at),
|
||||||
|
func.coalesce(func.sum(Video.duration_seconds), 0),
|
||||||
|
)
|
||||||
|
.where(Video.channel_id.in_(channel_ids))
|
||||||
|
.group_by(Video.channel_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.get("")
|
@router.get("")
|
||||||
def list_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
def list_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||||
rows = db.execute(
|
rows = db.execute(
|
||||||
select(Subscription, Channel)
|
select(Subscription, Channel)
|
||||||
.join(Channel, Channel.id == Subscription.channel_id)
|
.join(Channel, Channel.id == Subscription.channel_id)
|
||||||
.where(Subscription.user_id == user.id)
|
.where(Subscription.user_id == user.id)
|
||||||
.order_by(Subscription.priority.desc(), func.lower(Channel.title))
|
.order_by(Subscription.priority.desc(), collated(Channel.title, lang_for_user(user)))
|
||||||
).all()
|
).all()
|
||||||
|
|
||||||
# Per-channel aggregates over the shared catalog in ONE grouped pass: stored count, last
|
# Per-channel aggregates over the shared catalog (channel_aggregates), computed on read rather than
|
||||||
# upload, total duration. Cheap — a single grouped scan, so computed on read rather than
|
# denormalised: an Index Only Scan over a ~20 MB index, where it used to read every video's heap page
|
||||||
# denormalised (measure-first: reads aren't frequent/hot enough to warrant cached columns).
|
# (681 MB on prod, a 2–6 s first open once something else had pushed it out of memory, E31 S4d).
|
||||||
# (The Normal/Shorts/Live breakdown moved to the channel-detail page in R14, so it is no longer
|
# (The Normal/Shorts/Live breakdown moved to the channel-detail page in R14, so it is no longer
|
||||||
# computed here — the manager table dropped its N/S/L column.)
|
# computed here — the manager table dropped its N/S/L column.)
|
||||||
channel_ids = [c.id for _, c in rows]
|
channel_ids = [c.id for _, c in rows]
|
||||||
agg: dict[str, dict] = {}
|
agg: dict[str, dict] = {}
|
||||||
if channel_ids:
|
if channel_ids:
|
||||||
for cid, n, last_up, total_dur in db.execute(
|
for cid, n, last_up, total_dur in db.execute(channel_aggregates(channel_ids)).all():
|
||||||
select(
|
|
||||||
Video.channel_id,
|
|
||||||
func.count(Video.id),
|
|
||||||
func.max(Video.published_at),
|
|
||||||
func.coalesce(func.sum(Video.duration_seconds), 0),
|
|
||||||
)
|
|
||||||
.where(Video.channel_id.in_(channel_ids))
|
|
||||||
.group_by(Video.channel_id)
|
|
||||||
).all():
|
|
||||||
agg[cid] = {
|
agg[cid] = {
|
||||||
"stored": n,
|
"stored": n,
|
||||||
"last_video_at": iso(last_up),
|
"last_video_at": iso(last_up),
|
||||||
@@ -139,7 +148,7 @@ def _discovery_rows(db: Session, user: User) -> list:
|
|||||||
return db.execute(
|
return db.execute(
|
||||||
q.group_by(Channel.id).order_by(
|
q.group_by(Channel.id).order_by(
|
||||||
func.count(func.distinct(PlaylistItem.video_id)).desc(),
|
func.count(func.distinct(PlaylistItem.video_id)).desc(),
|
||||||
func.lower(Channel.title),
|
collated(Channel.title, lang_for_user(user)),
|
||||||
)
|
)
|
||||||
).all()
|
).all()
|
||||||
|
|
||||||
@@ -190,12 +199,16 @@ def discover_channels(
|
|||||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||||
YouTubeClient(db, user) as yt,
|
YouTubeClient(db, user) as yt,
|
||||||
):
|
):
|
||||||
apply_channel_details(db, yt.get_channels(need))
|
apply_channel_details(db, yt.get_channels(need), need)
|
||||||
db.commit()
|
db.commit()
|
||||||
# Match _discovery_rows' ORDER BY exactly: video-count DESC, then title (NULLs LAST,
|
# Match _discovery_rows' ORDER BY exactly: video-count DESC, then title in the reader's
|
||||||
# as Postgres orders them — coercing NULL→"" would wrongly float untitled channels up).
|
# alphabet (NULLs LAST, as Postgres orders them — coercing NULL→"" would wrongly float
|
||||||
|
# untitled channels up).
|
||||||
|
titles = [r[0].title for r in rows if r[0].title is not None]
|
||||||
|
alpha = {w: i for i, w in enumerate(sorted_words(db, titles, lang_for_user(user)))}
|
||||||
rows = sorted(
|
rows = sorted(
|
||||||
rows, key=lambda r: (-int(r[1]), r[0].title is None, (r[0].title or "").lower())
|
rows,
|
||||||
|
key=lambda r: (-int(r[1]), r[0].title is None, alpha.get(r[0].title or "", 0)),
|
||||||
)
|
)
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
log.warning(
|
log.warning(
|
||||||
@@ -296,17 +309,20 @@ def channel_detail(
|
|||||||
"""One channel's "About" detail for the channel page header/About tab, plus this user's
|
"""One channel's "About" detail for the channel page header/About tab, plus this user's
|
||||||
relationship to it (subscribed / explored) and how many of its videos are in the catalog.
|
relationship to it (subscribed / explored) and how many of its videos are in the catalog.
|
||||||
Lazily fills the About metadata (view count, join date, banner, links) the first time it's
|
Lazily fills the About metadata (view count, join date, banner, links) the first time it's
|
||||||
needed — a stub, or a channel last enriched before those fields existed (published_at is a
|
needed (a stub) and refreshes it once it is stale (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the
|
||||||
reliable sentinel: every real channel has a creation date). The enrich uses the API key
|
subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit,
|
||||||
(1 unit, cached after), so it's skipped for the quota-less demo account."""
|
cached after), so it's skipped for the quota-less demo account."""
|
||||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||||
if channel.published_at is None and not user.is_demo:
|
# No `published_at is None` sentinel: a terminated channel never gets one, so it re-fetched on
|
||||||
|
# every view; rows enriched before that field existed are long past DETAILS_MAX_AGE anyway.
|
||||||
|
stale = details_stale(channel, datetime.now(timezone.utc))
|
||||||
|
if stale and not user.is_demo:
|
||||||
try:
|
try:
|
||||||
with (
|
with (
|
||||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||||
YouTubeClient(db, user) as yt,
|
YouTubeClient(db, user) as yt,
|
||||||
):
|
):
|
||||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||||
db.commit()
|
db.commit()
|
||||||
channel = db.get(Channel, channel_id)
|
channel = db.get(Channel, channel_id)
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
@@ -412,17 +428,15 @@ def explore_channel(
|
|||||||
YouTubeClient(db, user) as yt,
|
YouTubeClient(db, user) as yt,
|
||||||
):
|
):
|
||||||
# Enrich for the About panel + the uploads playlist id (needed to ingest videos).
|
# Enrich for the About panel + the uploads playlist id (needed to ingest videos).
|
||||||
if (
|
# Stale, not "a field is missing": a terminated channel never gets an uploads playlist
|
||||||
channel.details_synced_at is None
|
# or a creation date, and asking again on every explore would spend quota on a miss.
|
||||||
or channel.uploads_playlist_id is None
|
if details_stale(channel, datetime.now(timezone.utc)):
|
||||||
or channel.published_at is None
|
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||||
):
|
|
||||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
|
||||||
db.commit()
|
db.commit()
|
||||||
channel = db.get(Channel, channel_id)
|
channel = db.get(Channel, channel_id)
|
||||||
result = explore_ingest_page(db, yt, channel, page_token)
|
result = explore_ingest_page(db, yt, channel, page_token)
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"channel_id": channel_id,
|
"channel_id": channel_id,
|
||||||
@@ -583,14 +597,14 @@ def subscribe(
|
|||||||
if "subscriptionDuplicate" in exc.reason.split(","):
|
if "subscriptionDuplicate" in exc.reason.split(","):
|
||||||
yt_sub_id = ""
|
yt_sub_id = ""
|
||||||
else:
|
else:
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
|
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
|
||||||
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
|
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
|
||||||
# render. Keep the client open (we're still inside the `with`); a metadata failure here
|
# render. Keep the client open (we're still inside the `with`); a metadata failure here
|
||||||
# must not block recording the subscription the user asked for.
|
# must not block recording the subscription the user asked for.
|
||||||
if channel.details_synced_at is None:
|
if channel.details_synced_at is None:
|
||||||
try:
|
try:
|
||||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
log.warning(
|
log.warning(
|
||||||
"subscribe: stub enrich failed: %s",
|
"subscribe: stub enrich failed: %s",
|
||||||
@@ -644,7 +658,7 @@ def unsubscribe(
|
|||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
|
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
|
||||||
# a transient "connection lost" gateway blip.
|
# a transient "connection lost" gateway blip.
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
db.delete(sub)
|
db.delete(sub)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"ok": True, "unsubscribed": channel_id}
|
return {"ok": True, "unsubscribed": channel_id}
|
||||||
@@ -658,7 +672,7 @@ def list_blocked(user: User = Depends(current_user), db: Session = Depends(get_d
|
|||||||
select(Channel)
|
select(Channel)
|
||||||
.join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
|
.join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
|
||||||
.where(BlockedChannel.user_id == user.id)
|
.where(BlockedChannel.user_id == user.id)
|
||||||
.order_by(func.lower(Channel.title))
|
.order_by(collated(Channel.title, lang_for_user(user)))
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from app.discover import hero as hero_mod
|
|||||||
from app.discover import pins, rails
|
from app.discover import pins, rails
|
||||||
from app.discover.budget import Budget, DEFAULT_BUDGET
|
from app.discover.budget import Budget, DEFAULT_BUDGET
|
||||||
from app.models import ComfortPin, User
|
from app.models import ComfortPin, User
|
||||||
|
from app.plex.cards import norm_lang
|
||||||
from app.routes._common import owned_or_404, not_found
|
from app.routes._common import owned_or_404, not_found
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/discover", tags=["discover"])
|
router = APIRouter(prefix="/api/discover", tags=["discover"])
|
||||||
@@ -45,6 +46,7 @@ class ReorderIn(BaseModel):
|
|||||||
def get_discover(
|
def get_discover(
|
||||||
budget: Budget = DEFAULT_BUDGET,
|
budget: Budget = DEFAULT_BUDGET,
|
||||||
seed: int = 0,
|
seed: int = 0,
|
||||||
|
lang: str = "en",
|
||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
@@ -52,13 +54,14 @@ def get_discover(
|
|||||||
|
|
||||||
`resume` and `shelf` are built once and handed to the hero, so the two rails the hero draws from
|
`resume` and `shelf` are built once and handed to the hero, so the two rails the hero draws from
|
||||||
are not queried twice for a single page load."""
|
are not queried twice for a single page load."""
|
||||||
resume = rails.continue_watching(db, user)
|
lang = norm_lang(lang)
|
||||||
|
resume = rails.continue_watching(db, user, lang=lang)
|
||||||
# Plex off → the shelf rail (and so the hero's comfort tier) goes YouTube-only like the rails
|
# Plex off → the shelf rail (and so the hero's comfort tier) goes YouTube-only like the rails
|
||||||
# beside it; the pins are kept and `GET /pins` still lists them (`pins.shelf`).
|
# beside it; the pins are kept and `GET /pins` still lists them (`pins.shelf`).
|
||||||
shelf = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
shelf = pins.shelf(db, user, include_plex=rails.plex_available(db), lang=lang)
|
||||||
return {
|
return {
|
||||||
"budget": budget,
|
"budget": budget,
|
||||||
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf),
|
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf, lang=lang),
|
||||||
"continue_watching": resume,
|
"continue_watching": resume,
|
||||||
"comfort": shelf,
|
"comfort": shelf,
|
||||||
}
|
}
|
||||||
@@ -80,6 +83,7 @@ def get_pick(
|
|||||||
seed: int = 0,
|
seed: int = 0,
|
||||||
exclude: str = "",
|
exclude: str = "",
|
||||||
start: int = 0,
|
start: int = 0,
|
||||||
|
lang: str = "en",
|
||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
@@ -99,18 +103,25 @@ def get_pick(
|
|||||||
return {
|
return {
|
||||||
"budget": budget,
|
"budget": budget,
|
||||||
"hero": hero_mod.pick(
|
"hero": hero_mod.pick(
|
||||||
db, user, budget, seed, exclude=set(list(skip)[:MAX_EXCLUDE]), start=start
|
db,
|
||||||
|
user,
|
||||||
|
budget,
|
||||||
|
seed,
|
||||||
|
exclude=set(list(skip)[:MAX_EXCLUDE]),
|
||||||
|
start=start,
|
||||||
|
lang=norm_lang(lang),
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/pins")
|
@router.get("/pins")
|
||||||
def list_pins(
|
def list_pins(
|
||||||
|
lang: str = "en",
|
||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""The shelf on its own — for the shelf editor, which needs it without the rest of the page."""
|
"""The shelf on its own — for the shelf editor, which needs it without the rest of the page."""
|
||||||
return {"items": pins.shelf(db, user)}
|
return {"items": pins.shelf(db, user, lang=norm_lang(lang))}
|
||||||
|
|
||||||
|
|
||||||
@router.post("/pins")
|
@router.post("/pins")
|
||||||
@@ -159,7 +170,7 @@ def add_pin(
|
|||||||
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
||||||
# item), which trips the FK. Semantically that is the same "no such thing" the pre-check
|
# item), which trips the FK. Semantically that is the same "no such thing" the pre-check
|
||||||
# returns above — re-raising handed the user a 500 they could do nothing with.
|
# returns above — re-raising handed the user a 500 they could do nothing with.
|
||||||
raise not_found("item")
|
raise not_found("item") from None
|
||||||
return {"ok": True, "id": winner.id, "created": False}
|
return {"ok": True, "id": winner.id, "created": False}
|
||||||
return {"ok": True, "id": pin.id, "created": True}
|
return {"ok": True, "id": pin.id, "created": True}
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ from pydantic import BaseModel
|
|||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.mailtext import lang_for_user
|
||||||
|
from app.collation import collated
|
||||||
from app import audit, sysconfig
|
from app import audit, sysconfig
|
||||||
from app.auth import admin_user, require_human
|
from app.auth import admin_user, require_human
|
||||||
from app.config import settings
|
from app.config import settings
|
||||||
@@ -320,7 +322,7 @@ def set_credential(
|
|||||||
try:
|
try:
|
||||||
cleaned = dlcreds.validate_cookies(body.cookies)
|
cleaned = dlcreds.validate_cookies(body.cookies)
|
||||||
except dlcreds.InvalidCookies as e:
|
except dlcreds.InvalidCookies as e:
|
||||||
raise HTTPException(status_code=400, detail={"code": "cookies", "reason": str(e)})
|
raise HTTPException(status_code=400, detail={"code": "cookies", "reason": str(e)}) from e
|
||||||
dlcreds.upsert_credential(db, user.id, host_key, cleaned)
|
dlcreds.upsert_credential(db, user.id, host_key, cleaned)
|
||||||
audit.record(
|
audit.record(
|
||||||
db,
|
db,
|
||||||
@@ -405,7 +407,7 @@ def enqueue_download(
|
|||||||
try:
|
try:
|
||||||
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
|
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
|
||||||
except quota.QuotaExceeded as e:
|
except quota.QuotaExceeded as e:
|
||||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
|
||||||
return _serialize_job(db, job)
|
return _serialize_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
@@ -518,7 +520,7 @@ def upload_download(
|
|||||||
display_name=upload.display_name,
|
display_name=upload.display_name,
|
||||||
)
|
)
|
||||||
except quota.QuotaExceeded as e:
|
except quota.QuotaExceeded as e:
|
||||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
|
||||||
return _serialize_job(db, job)
|
return _serialize_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
@@ -545,9 +547,9 @@ def enqueue_edit(
|
|||||||
try:
|
try:
|
||||||
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
|
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
|
||||||
except quota.QuotaExceeded as e:
|
except quota.QuotaExceeded as e:
|
||||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
|
||||||
except service.EditError as e:
|
except service.EditError as e:
|
||||||
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason})
|
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason}) from e
|
||||||
return _serialize_job(db, job)
|
return _serialize_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
@@ -917,7 +919,14 @@ def share_download(
|
|||||||
user_id=recipient.id,
|
user_id=recipient.id,
|
||||||
type="download_shared",
|
type="download_shared",
|
||||||
title=(job.display_name or "A download") + " was shared with you",
|
title=(job.display_name or "A download") + " was shared with you",
|
||||||
data={"kind": "download_shared", "from": user.email, "job_id": job_id},
|
# The client names the notice in the reader's language from `data` (E30); the title is the
|
||||||
|
# English fallback only.
|
||||||
|
data={
|
||||||
|
"kind": "download_shared",
|
||||||
|
"from": user.email,
|
||||||
|
"job_id": job_id,
|
||||||
|
"title": job.display_name,
|
||||||
|
},
|
||||||
commit=False,
|
commit=False,
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
@@ -981,7 +990,7 @@ def share_recipients(
|
|||||||
db.execute(
|
db.execute(
|
||||||
select(User)
|
select(User)
|
||||||
.where(User.id != user.id, *messageable_clauses())
|
.where(User.id != user.id, *messageable_clauses())
|
||||||
.order_by(func.lower(User.email))
|
.order_by(collated(User.email, lang_for_user(user)))
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
|
|||||||
+31
-12
@@ -10,6 +10,7 @@ from sqlalchemy.orm.exc import StaleDataError
|
|||||||
from app import quota
|
from app import quota
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
|
from app.mailtext import lang_for_user
|
||||||
from app.feedquery import (
|
from app.feedquery import (
|
||||||
decode_cursor,
|
decode_cursor,
|
||||||
encode_cursor,
|
encode_cursor,
|
||||||
@@ -18,8 +19,11 @@ from app.feedquery import (
|
|||||||
filtered_query,
|
filtered_query,
|
||||||
keyset_predicate,
|
keyset_predicate,
|
||||||
order_by,
|
order_by,
|
||||||
|
rank_window_start,
|
||||||
serialize,
|
serialize,
|
||||||
sort_keys,
|
sort_keys,
|
||||||
|
windowed_rank,
|
||||||
|
windowed_relevance_rows,
|
||||||
)
|
)
|
||||||
from app.models import (
|
from app.models import (
|
||||||
POSITION_MAX_S,
|
POSITION_MAX_S,
|
||||||
@@ -108,30 +112,45 @@ def get_feed(
|
|||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
query, rank_expr = filtered_query(db, user, **params)
|
query, rank_expr = filtered_query(db, user, **params)
|
||||||
keys = sort_keys(sort, seed, rank_expr)
|
lang = lang_for_user(user)
|
||||||
|
keys = sort_keys(sort, seed, rank_expr, lang)
|
||||||
# Expose each key column on the row so we can build the next cursor from the last item.
|
|
||||||
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
|
||||||
query = query.order_by(*order_by(keys))
|
|
||||||
|
|
||||||
|
vals, cursor_id, window = None, None, None
|
||||||
if cursor:
|
if cursor:
|
||||||
# The read-model raises a plain ValueError on a malformed cursor; translate it to the
|
# The read-model raises a plain ValueError on a malformed cursor; translate it to the
|
||||||
# HTTP 400 here so feedquery stays web-framework-free.
|
# HTTP 400 here so feedquery stays web-framework-free.
|
||||||
try:
|
try:
|
||||||
vals, cursor_id = decode_cursor(keys, cursor)
|
vals, cursor_id, window = decode_cursor(keys, cursor)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
# `from None`: the ValueError is an expected bad-input signal, not an error to chain.
|
# `from None`: the ValueError is an expected bad-input signal, not an error to chain.
|
||||||
raise refused(400, "invalid_cursor") from None
|
raise refused(400, "invalid_cursor") from None
|
||||||
for k, v in zip(keys, vals):
|
ranked = sort == "relevance" and rank_expr is not None
|
||||||
k["value"] = v
|
if ranked and not cursor:
|
||||||
query = query.where(keyset_predicate(keys, cursor_id))
|
window = rank_window_start(db, query)
|
||||||
|
if ranked and window is not None:
|
||||||
|
keys = sort_keys(sort, seed, windowed_rank(rank_expr, window), lang)
|
||||||
|
|
||||||
rows = db.execute(query.limit(limit + 1)).all()
|
# Expose each key column on the row so we can build the next cursor from the last item.
|
||||||
|
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
||||||
|
if vals is not None:
|
||||||
|
for k, v in zip(keys, vals, strict=True):
|
||||||
|
k["value"] = v
|
||||||
|
if ranked and window is not None:
|
||||||
|
rows = windowed_relevance_rows(db, query, keys, window, cursor_id, limit + 1)
|
||||||
|
else:
|
||||||
|
query = query.order_by(*order_by(keys))
|
||||||
|
if cursor_id is not None:
|
||||||
|
query = query.where(keyset_predicate(keys, cursor_id))
|
||||||
|
rows = db.execute(query.limit(limit + 1)).all()
|
||||||
has_more = len(rows) > limit
|
has_more = len(rows) > limit
|
||||||
page = rows[:limit]
|
page = rows[:limit]
|
||||||
return {
|
return {
|
||||||
"items": [serialize(r) for r in page],
|
"items": [serialize(r) for r in page],
|
||||||
"next_cursor": encode_cursor(keys, page[-1]) if (has_more and page) else None,
|
"next_cursor": (
|
||||||
|
encode_cursor(keys, page[-1], window if ranked else None)
|
||||||
|
if (has_more and page)
|
||||||
|
else None
|
||||||
|
),
|
||||||
"has_more": has_more,
|
"has_more": has_more,
|
||||||
"limit": limit,
|
"limit": limit,
|
||||||
}
|
}
|
||||||
@@ -412,7 +431,7 @@ def get_video_detail(
|
|||||||
):
|
):
|
||||||
items = yt.get_videos([video_id])
|
items = yt.get_videos([video_id])
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
if not items:
|
if not items:
|
||||||
raise not_found("video")
|
raise not_found("video")
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ from pydantic import BaseModel
|
|||||||
from sqlalchemy import and_, case, delete, func, or_, select
|
from sqlalchemy import and_, case, delete, func, or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.mailtext import lang_for_user
|
||||||
|
from app.collation import collated
|
||||||
from app.auth import require_human, resolved_user_id
|
from app.auth import require_human, resolved_user_id
|
||||||
from app.db import SessionLocal, get_db
|
from app.db import SessionLocal, get_db
|
||||||
from app.models import Message, MessageKey, User
|
from app.models import Message, MessageKey, User
|
||||||
@@ -92,13 +94,31 @@ def _system_partner() -> dict:
|
|||||||
return {"id": SYSTEM_PARTNER_ID, "name": SYSTEM_NAME, "avatar_url": None}
|
return {"id": SYSTEM_PARTNER_ID, "name": SYSTEM_NAME, "avatar_url": None}
|
||||||
|
|
||||||
|
|
||||||
def _serialize_msg(m: Message) -> dict:
|
# The welcome's opening line in every language — how a stored copy is recognised. It was rendered
|
||||||
|
# once, in the language of the day it was created, so an account that later switched language kept
|
||||||
|
# reading it in the old one (E31 S1: the owner's, in English). Recognised here, it is served in the
|
||||||
|
# reader's language instead. Derived from WELCOME itself, so a change to an OPENING line, or a
|
||||||
|
# language taken out of WELCOME, orphans the copies already stored: they are served as stored. The
|
||||||
|
# one retired language (German, 2026-07) left none on prod (checked 2026-10-02).
|
||||||
|
_WELCOME_OPENINGS = {text.partition("\n")[0] for text in WELCOME.values()}
|
||||||
|
|
||||||
|
|
||||||
|
def _system_body(body: str | None, lang: str | None) -> str | None:
|
||||||
|
"""A system message's text for a reader of `lang`: the welcome in their language, any other as
|
||||||
|
stored."""
|
||||||
|
if body is None or lang is None or body.partition("\n")[0] not in _WELCOME_OPENINGS:
|
||||||
|
return body
|
||||||
|
return WELCOME.get(lang, WELCOME["en"])
|
||||||
|
|
||||||
|
|
||||||
|
def _serialize_msg(m: Message, lang: str | None = None) -> dict:
|
||||||
|
"""`lang`: the reader's language, for a system message's text (a user message is ciphertext)."""
|
||||||
return {
|
return {
|
||||||
"id": m.id,
|
"id": m.id,
|
||||||
"kind": m.kind,
|
"kind": m.kind,
|
||||||
"sender_id": m.sender_id,
|
"sender_id": m.sender_id,
|
||||||
"recipient_id": m.recipient_id,
|
"recipient_id": m.recipient_id,
|
||||||
"body": m.body, # only set for system messages
|
"body": _system_body(m.body, lang) if m.kind == "system" else m.body,
|
||||||
"ciphertext": m.ciphertext,
|
"ciphertext": m.ciphertext,
|
||||||
"iv": m.iv,
|
"iv": m.iv,
|
||||||
"read_at": iso(m.read_at),
|
"read_at": iso(m.read_at),
|
||||||
@@ -122,7 +142,7 @@ def ensure_welcome(db: Session, user: User) -> None:
|
|||||||
db.add(m)
|
db.add(m)
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(m)
|
db.refresh(m)
|
||||||
manager.push(user.id, {"type": "message", "message": _serialize_msg(m)})
|
manager.push(user.id, {"type": "message", "message": _serialize_msg(m, lang_for_user(user))})
|
||||||
|
|
||||||
|
|
||||||
# --- E2EE key directory / blob store -----------------------------------------
|
# --- E2EE key directory / blob store -----------------------------------------
|
||||||
@@ -246,7 +266,7 @@ def list_recipients(
|
|||||||
select(User)
|
select(User)
|
||||||
.join(MessageKey, MessageKey.user_id == User.id)
|
.join(MessageKey, MessageKey.user_id == User.id)
|
||||||
.where(User.id != user.id, *messageable_clauses())
|
.where(User.id != user.id, *messageable_clauses())
|
||||||
.order_by(func.lower(func.coalesce(User.display_name, User.email)))
|
.order_by(collated(func.coalesce(User.display_name, User.email), lang_for_user(user)))
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
@@ -311,7 +331,7 @@ def list_conversations(user: User = Depends(require_human), db: Session = Depend
|
|||||||
items.append(
|
items.append(
|
||||||
{
|
{
|
||||||
"partner": partner,
|
"partner": partner,
|
||||||
"last_message": _serialize_msg(m),
|
"last_message": _serialize_msg(m, lang_for_user(user)),
|
||||||
"unread": int(unread.get(pid, 0)),
|
"unread": int(unread.get(pid, 0)),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
@@ -364,7 +384,7 @@ def get_thread(
|
|||||||
# refreshes live instead of waiting for the next 20s poll. The client re-fetches the
|
# refreshes live instead of waiting for the next 20s poll. The client re-fetches the
|
||||||
# count on this signal, so no number is carried here.
|
# count on this signal, so no number is carried here.
|
||||||
manager.push(user.id, {"type": "unread"})
|
manager.push(user.id, {"type": "unread"})
|
||||||
return {"partner": partner, "items": [_serialize_msg(m) for m in msgs]}
|
return {"partner": partner, "items": [_serialize_msg(m, lang_for_user(user)) for m in msgs]}
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
@router.post("")
|
||||||
|
|||||||
+129
-72
@@ -4,9 +4,8 @@ mirror the user's own YouTube playlists in (`/sync-youtube`) and push/revert loc
|
|||||||
out (`/{id}/push-plan`, `/{id}/revert-youtube`), behind the optional write scope."""
|
out (`/{id}/push-plan`, `/{id}/revert-youtube`), behind the optional write scope."""
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException
|
from fastapi import APIRouter, Depends, HTTPException
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel, Field
|
||||||
from sqlalchemy import func, select, and_
|
from sqlalchemy import func, select, and_
|
||||||
from sqlalchemy.exc import IntegrityError
|
|
||||||
from sqlalchemy.orm import Session, aliased
|
from sqlalchemy.orm import Session, aliased
|
||||||
|
|
||||||
from app import quota
|
from app import quota
|
||||||
@@ -15,14 +14,18 @@ from app.db import get_db
|
|||||||
from app.feedquery import feed_columns, serialize
|
from app.feedquery import feed_columns, serialize
|
||||||
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
|
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
|
||||||
from app.sync.playlists import (
|
from app.sync.playlists import (
|
||||||
|
PlaylistGone,
|
||||||
|
lock_playlists,
|
||||||
merge_playlist,
|
merge_playlist,
|
||||||
plan_merge,
|
plan_merge,
|
||||||
recompute_dirty,
|
recompute_dirty,
|
||||||
|
pull_user_playlists,
|
||||||
repull_playlist,
|
repull_playlist,
|
||||||
sync_user_playlists,
|
sync_claim,
|
||||||
|
user_sync_claim,
|
||||||
)
|
)
|
||||||
from app.ratelimit import RateLimiter
|
from app.ratelimit import RateLimiter
|
||||||
from app.routes._common import get_or_404, owned_or_404, youtube_refusal
|
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||||
from app.youtube.client import YouTubeClient, YouTubeError
|
from app.youtube.client import YouTubeClient, YouTubeError
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/playlists", tags=["playlists"])
|
router = APIRouter(prefix="/api/playlists", tags=["playlists"])
|
||||||
@@ -79,7 +82,10 @@ def _next_playlist_position(db: Session, user_id: int) -> int:
|
|||||||
|
|
||||||
|
|
||||||
def _next_item_position(db: Session, playlist_id: int) -> int:
|
def _next_item_position(db: Session, playlist_id: int) -> int:
|
||||||
"""Append position for a new item at the end of a playlist."""
|
"""Append position for a new item at the end of a playlist. Locks the playlist row first: two
|
||||||
|
adds (two tabs, the rail and the page) otherwise both read the same max and land on one
|
||||||
|
position, and the list's order between them is then arbitrary."""
|
||||||
|
lock_playlists(db, playlist_id)
|
||||||
return (
|
return (
|
||||||
db.scalar(
|
db.scalar(
|
||||||
select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
|
select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
|
||||||
@@ -92,7 +98,10 @@ def _next_item_position(db: Session, playlist_id: int) -> int:
|
|||||||
|
|
||||||
def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||||
"""Append a video to a playlist if not already present (idempotent). Returns whether it
|
"""Append a video to a playlist if not already present (idempotent). Returns whether it
|
||||||
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later).
|
||||||
|
Locked BEFORE the presence check: two adds of one video both saw "absent" and the second one's
|
||||||
|
insert hit uq_playlist_video as a 500."""
|
||||||
|
lock_playlists(db, pl.id)
|
||||||
exists = db.scalar(
|
exists = db.scalar(
|
||||||
select(PlaylistItem).where(
|
select(PlaylistItem).where(
|
||||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||||
@@ -113,6 +122,7 @@ def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) ->
|
|||||||
def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||||
"""Remove a video from a playlist if present (idempotent). Returns whether it was removed.
|
"""Remove a video from a playlist if present (idempotent). Returns whether it was removed.
|
||||||
`mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
`mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
||||||
|
lock_playlists(db, pl.id)
|
||||||
item = db.scalar(
|
item = db.scalar(
|
||||||
select(PlaylistItem).where(
|
select(PlaylistItem).where(
|
||||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||||
@@ -321,13 +331,18 @@ def sync_youtube(
|
|||||||
Also runs automatically on the scheduler; this is the manual 'sync now'."""
|
Also runs automatically on the scheduler; this is the manual 'sync now'."""
|
||||||
if not has_read_scope(user):
|
if not has_read_scope(user):
|
||||||
raise _refused(403, "youtube_read_scope")
|
raise _refused(403, "youtube_read_scope")
|
||||||
_check_pull_cooldown(user)
|
with user_sync_claim(db, user.id) as free:
|
||||||
try:
|
if not free:
|
||||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
# Another run (the scheduler, another tab) is pulling right now: "0 synced" would lie.
|
||||||
return {"ok": True, **sync_user_playlists(db, user)}
|
raise _refused(409, "sync_all_busy")
|
||||||
except YouTubeError as exc:
|
# After the claim, like the re-pull's: a refused attempt must not spend one of the pulls.
|
||||||
db.rollback()
|
_check_pull_cooldown(user)
|
||||||
raise youtube_refusal(exc)
|
try:
|
||||||
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||||
|
return {"ok": True, **pull_user_playlists(db, user)}
|
||||||
|
except YouTubeError as exc:
|
||||||
|
db.rollback()
|
||||||
|
raise youtube_refusal(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{playlist_id}/revert-youtube")
|
@router.post("/{playlist_id}/revert-youtube")
|
||||||
@@ -344,13 +359,20 @@ def revert_youtube(
|
|||||||
raise _refused(400, "not_on_youtube")
|
raise _refused(400, "not_on_youtube")
|
||||||
if not has_read_scope(user):
|
if not has_read_scope(user):
|
||||||
raise _refused(403, "youtube_read_scope")
|
raise _refused(403, "youtube_read_scope")
|
||||||
_check_pull_cooldown(user)
|
with sync_claim(db, pl.id) as free:
|
||||||
try:
|
if not free:
|
||||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
raise _refused(409, "sync_busy")
|
||||||
return {"ok": True, **repull_playlist(db, user, pl)}
|
# The cooldown is spent only by a pull that runs: checked after the claim, so clicking
|
||||||
except YouTubeError as exc:
|
# Revert while a push is in flight does not use up the three pulls per ten minutes.
|
||||||
db.rollback()
|
_check_pull_cooldown(user)
|
||||||
raise youtube_refusal(exc)
|
try:
|
||||||
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||||
|
return {"ok": True, **repull_playlist(db, user, pl)}
|
||||||
|
except PlaylistGone:
|
||||||
|
raise not_found("playlist") from None
|
||||||
|
except YouTubeError as exc:
|
||||||
|
db.rollback()
|
||||||
|
raise youtube_refusal(exc, playlist=True) from exc
|
||||||
|
|
||||||
|
|
||||||
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||||
@@ -376,7 +398,7 @@ def push_plan(
|
|||||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||||
plan = plan_merge(db, user, pl)
|
plan = plan_merge(db, user, pl)
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc, playlist=True) from exc
|
||||||
plan["remaining_today"] = quota.remaining_today(db)
|
plan["remaining_today"] = quota.remaining_today(db)
|
||||||
plan["affordable"] = plan["units_estimate"] <= plan["remaining_today"]
|
plan["affordable"] = plan["units_estimate"] <= plan["remaining_today"]
|
||||||
return plan
|
return plan
|
||||||
@@ -392,25 +414,32 @@ def push(
|
|||||||
neither side loses an item). Refuses if the estimated quota exceeds what's left today, so a
|
neither side loses an item). Refuses if the estimated quota exceeds what's left today, so a
|
||||||
big sync can't strand half-applied. Write scope required (require_write_scope)."""
|
big sync can't strand half-applied. Write scope required (require_write_scope)."""
|
||||||
pl = _pushable(db, user, playlist_id)
|
pl = _pushable(db, user, playlist_id)
|
||||||
try:
|
# One sync of a list at a time, across tabs, devices and the scheduler (`sync_claim`): the
|
||||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
# client's own in-flight flag only covers the menus of one page.
|
||||||
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
with sync_claim(db, pl.id) as free:
|
||||||
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
if not free:
|
||||||
# window. A fresh export (no yt_playlist_id) reads nothing.
|
raise _refused(409, "sync_busy")
|
||||||
live = None
|
try:
|
||||||
if pl.yt_playlist_id:
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||||
with YouTubeClient(db, user) as yt:
|
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||||
live = (
|
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
||||||
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||||
yt.get_playlist_snippet(pl.yt_playlist_id),
|
live = None
|
||||||
)
|
if pl.yt_playlist_id:
|
||||||
plan = plan_merge(db, user, pl, live=live)
|
with YouTubeClient(db, user) as yt:
|
||||||
if plan["units_estimate"] > quota.remaining_today(db):
|
live = (
|
||||||
raise _refused(429, "quota_exceeded")
|
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||||
return {"ok": True, **merge_playlist(db, user, pl, live=live)}
|
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||||
except YouTubeError as exc:
|
)
|
||||||
db.rollback()
|
plan = plan_merge(db, user, pl, live=live)
|
||||||
raise youtube_refusal(exc)
|
if plan["units_estimate"] > quota.remaining_today(db):
|
||||||
|
raise _refused(429, "quota_exceeded")
|
||||||
|
return {"ok": True, **merge_playlist(db, user, pl, live=live)}
|
||||||
|
except PlaylistGone:
|
||||||
|
raise not_found("playlist") from None
|
||||||
|
except YouTubeError as exc:
|
||||||
|
db.rollback()
|
||||||
|
raise youtube_refusal(exc, playlist=True) from exc
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{playlist_id}")
|
@router.get("/{playlist_id}")
|
||||||
@@ -427,7 +456,8 @@ def get_playlist(
|
|||||||
.join(Channel, Channel.id == Video.channel_id)
|
.join(Channel, Channel.id == Video.channel_id)
|
||||||
.outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id))
|
.outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id))
|
||||||
.where(PlaylistItem.playlist_id == pl.id)
|
.where(PlaylistItem.playlist_id == pl.id)
|
||||||
.order_by(PlaylistItem.position)
|
# `id` breaks a position tie the same way on every read (the renumbering add does too).
|
||||||
|
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||||
).all()
|
).all()
|
||||||
return {
|
return {
|
||||||
"id": pl.id,
|
"id": pl.id,
|
||||||
@@ -451,7 +481,8 @@ def rename_playlist(
|
|||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
# Locked: the new name moves `dirty`, which the scheduled mirror refresh reads under this lock.
|
||||||
|
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||||
if "name" in body.model_fields_set:
|
if "name" in body.model_fields_set:
|
||||||
name = (body.name or "").strip()
|
name = (body.name or "").strip()
|
||||||
if not name:
|
if not name:
|
||||||
@@ -491,33 +522,65 @@ def delete_playlist(
|
|||||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||||
if pl.kind == "watch_later":
|
if pl.kind == "watch_later":
|
||||||
raise _refused(400, "watch_later_undeletable")
|
raise _refused(400, "watch_later_undeletable")
|
||||||
if on_youtube:
|
# Not in the middle of a sync of it (`sync_claim`): a push that read the list before the delete
|
||||||
if not pl.yt_playlist_id:
|
# and wrote YouTube after it planned from no items at all, so it emptied the YouTube playlist a
|
||||||
raise _refused(400, "not_on_youtube")
|
# local-only delete means to keep. The push checks for a vanished list too (`PlaylistGone`);
|
||||||
if not has_write_scope(user):
|
# this refuses the delete while a sync runs rather than race it.
|
||||||
raise _refused(403, "youtube_editing_off")
|
with sync_claim(db, pl.id) as free:
|
||||||
try:
|
if not free:
|
||||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
raise _refused(409, "sync_busy")
|
||||||
with YouTubeClient(db, user) as yt:
|
if on_youtube:
|
||||||
yt.delete_playlist(pl.yt_playlist_id)
|
if not pl.yt_playlist_id:
|
||||||
except YouTubeError as exc:
|
raise _refused(400, "not_on_youtube")
|
||||||
raise youtube_refusal(exc)
|
if not has_write_scope(user):
|
||||||
db.delete(pl) # items cascade
|
raise _refused(403, "youtube_editing_off")
|
||||||
db.commit()
|
try:
|
||||||
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
||||||
|
with YouTubeClient(db, user) as yt:
|
||||||
|
yt.delete_playlist(pl.yt_playlist_id)
|
||||||
|
except YouTubeError as exc:
|
||||||
|
raise youtube_refusal(exc, playlist=True) from exc
|
||||||
|
# The playlist first, as every item write: the cascade deletes the items before the list row.
|
||||||
|
lock_playlists(db, pl.id)
|
||||||
|
db.delete(pl) # items cascade
|
||||||
|
db.commit()
|
||||||
return {"deleted": playlist_id}
|
return {"deleted": playlist_id}
|
||||||
|
|
||||||
|
|
||||||
|
class AddItemIn(VideoRefIn):
|
||||||
|
# Where the video lands among the list's items, in their stored order (0 = first). Omitted, it
|
||||||
|
# is appended. The list page's "Undo" of a removal puts the row back where it was with this —
|
||||||
|
# ONE request, because append-then-reorder leaves it at the end whenever the second call fails.
|
||||||
|
position: int | None = Field(default=None, ge=0)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{playlist_id}/items")
|
@router.post("/{playlist_id}/items")
|
||||||
def add_item(
|
def add_item(
|
||||||
playlist_id: int,
|
playlist_id: int,
|
||||||
body: VideoRefIn,
|
body: AddItemIn,
|
||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||||
video_id = body.video_id
|
video_id = body.video_id
|
||||||
get_or_404(db, Video, video_id, what="video")
|
get_or_404(db, Video, video_id, what="video")
|
||||||
_add_item(db, pl, video_id, mark_dirty=True)
|
added = _add_item(db, pl, video_id, mark_dirty=False)
|
||||||
|
if added and body.position is not None:
|
||||||
|
# Renumber the whole list with the new row at `position` (clamped to the end): positions
|
||||||
|
# can have gaps after removals, so "index N" is only meaningful over the ordered rows.
|
||||||
|
rows = list(
|
||||||
|
db.execute(
|
||||||
|
select(PlaylistItem)
|
||||||
|
.where(PlaylistItem.playlist_id == pl.id)
|
||||||
|
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||||
|
).scalars()
|
||||||
|
)
|
||||||
|
new = next(r for r in rows if r.video_id == video_id)
|
||||||
|
rows.remove(new)
|
||||||
|
rows.insert(min(body.position, len(rows)), new)
|
||||||
|
for pos, row in enumerate(rows):
|
||||||
|
row.position = pos
|
||||||
|
recompute_dirty(db, pl)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||||
|
|
||||||
@@ -562,6 +625,8 @@ def move_item(
|
|||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
status_code=400, detail={"code": "playlist_move", "reason": "same_playlist"}
|
status_code=400, detail={"code": "playlist_move", "reason": "same_playlist"}
|
||||||
)
|
)
|
||||||
|
# Both lists, in id order: a move A→B beside a move B→A would otherwise lock crosswise.
|
||||||
|
lock_playlists(db, src.id, dst.id)
|
||||||
item = db.scalar(
|
item = db.scalar(
|
||||||
select(PlaylistItem).where(
|
select(PlaylistItem).where(
|
||||||
PlaylistItem.playlist_id == src.id, PlaylistItem.video_id == video_id
|
PlaylistItem.playlist_id == src.id, PlaylistItem.video_id == video_id
|
||||||
@@ -581,20 +646,11 @@ def move_item(
|
|||||||
if already is None:
|
if already is None:
|
||||||
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
|
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
|
||||||
# survives the move, and there is no window where the video belongs to neither list. The
|
# survives the move, and there is no window where the video belongs to neither list. The
|
||||||
# check above is not a lock: a second tab's add or the scheduler's YouTube merge can insert
|
# check above holds: both lists are locked, and an insert into the target (another tab's
|
||||||
# the same (playlist, video) in between, and uq_playlist_video then refuses the re-parent.
|
# add, a YouTube merge) waits for the lock — its foreign key needs the target row.
|
||||||
# That is the already-present outcome arriving late, not a failure — answer it the same way
|
item.playlist_id = dst.id
|
||||||
# instead of a raw 500.
|
item.position = _next_item_position(db, dst.id)
|
||||||
try:
|
dst_changed = True
|
||||||
with db.begin_nested():
|
|
||||||
item.playlist_id = dst.id
|
|
||||||
item.position = _next_item_position(db, dst.id)
|
|
||||||
db.flush()
|
|
||||||
dst_changed = True
|
|
||||||
# silent-ok: the conflict IS the answer — the target already holds the video, which is the
|
|
||||||
# idempotent outcome; the savepoint put the row back in the source list, dropped below.
|
|
||||||
except IntegrityError:
|
|
||||||
pass
|
|
||||||
if not dst_changed:
|
if not dst_changed:
|
||||||
db.delete(item)
|
db.delete(item)
|
||||||
# The source always changed; the target only when the row landed in it. `_touch` on a target
|
# The source always changed; the target only when the row landed in it. `_touch` on a target
|
||||||
@@ -619,7 +675,8 @@ def reorder_items(
|
|||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
# Locked: a concurrent add between reading the items and writing the order would collide.
|
||||||
|
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||||
order = body.video_ids
|
order = body.video_ids
|
||||||
items = {
|
items = {
|
||||||
it.video_id: it
|
it.video_id: it
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app import logs
|
from app import logs
|
||||||
from app.models import PlexItem
|
from app.models import PlexItem
|
||||||
|
from app.plex.cards import norm_lang as _norm_lang # noqa: F401 — the package's historic name
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
from app.routes._common import not_found
|
from app.routes._common import not_found
|
||||||
|
|
||||||
@@ -15,8 +16,3 @@ def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
|||||||
if it is None:
|
if it is None:
|
||||||
raise not_found("plex_item")
|
raise not_found("plex_item")
|
||||||
return it
|
return it
|
||||||
|
|
||||||
|
|
||||||
def _norm_lang(lang: str | None) -> str:
|
|
||||||
"""The overlay has an EN base + an optional HU twin; anything other than `hu` serves EN."""
|
|
||||||
return "hu" if lang == "hu" else "en"
|
|
||||||
|
|||||||
@@ -10,10 +10,13 @@ from sqlalchemy import (
|
|||||||
func,
|
func,
|
||||||
literal,
|
literal,
|
||||||
null,
|
null,
|
||||||
|
or_,
|
||||||
select,
|
select,
|
||||||
)
|
)
|
||||||
from sqlalchemy.orm import Session, aliased
|
from sqlalchemy.orm import Session, aliased
|
||||||
|
|
||||||
|
from app.mailtext import lang_for_user
|
||||||
|
from app.collation import collated, sorted_words
|
||||||
from app import sysconfig
|
from app import sysconfig
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
@@ -25,7 +28,7 @@ from app.models import (
|
|||||||
User,
|
User,
|
||||||
)
|
)
|
||||||
from app.plex import rollup
|
from app.plex import rollup
|
||||||
from app.plex.cards import episode_card
|
from app.plex.cards import display_title, episode_card
|
||||||
from app.plex.query import (
|
from app.plex.query import (
|
||||||
FIELD_COLUMNS,
|
FIELD_COLUMNS,
|
||||||
PLAYBACK_CLASSES,
|
PLAYBACK_CLASSES,
|
||||||
@@ -54,14 +57,15 @@ from app.utils import to_tsquery_str
|
|||||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||||
|
|
||||||
|
|
||||||
def sort_column(u, sort: str):
|
def sort_column(u, sort: str, lang: str = "en"):
|
||||||
"""The browse grid's sort key -> orderable column on the union subquery. Module-level (not inline
|
"""The browse grid's sort key -> orderable column on the union subquery. Module-level (not inline
|
||||||
in the handler) so a test can pin its vocabulary to `PLEX_SORT_KEYS` — a key the Sort control
|
in the handler) so a test can pin its vocabulary to `PLEX_SORT_KEYS` — a key the Sort control
|
||||||
offers but this cannot order would quietly fall back to `added_at` and look like a broken control.
|
offers but this cannot order would quietly fall back to `added_at` and look like a broken control.
|
||||||
An unknown key (a stale bookmark, a hand-edited URL) deliberately keeps that fallback."""
|
An unknown key (a stale bookmark, a hand-edited URL) deliberately keeps that fallback. The title sorts
|
||||||
|
in the reader's alphabet (`collation`)."""
|
||||||
return {
|
return {
|
||||||
"added": u.c.added_at,
|
"added": u.c.added_at,
|
||||||
"title": func.lower(u.c.title),
|
"title": collated(u.c.title, lang),
|
||||||
"year": u.c.year,
|
"year": u.c.year,
|
||||||
"rating": u.c.rating,
|
"rating": u.c.rating,
|
||||||
"duration": u.c.duration_s,
|
"duration": u.c.duration_s,
|
||||||
@@ -78,7 +82,7 @@ def list_libraries(user: User = Depends(current_user), db: Session = Depends(get
|
|||||||
db.execute(
|
db.execute(
|
||||||
select(PlexLibrary)
|
select(PlexLibrary)
|
||||||
.where(PlexLibrary.overlay_of.is_(None))
|
.where(PlexLibrary.overlay_of.is_(None))
|
||||||
.order_by(PlexLibrary.kind.desc(), PlexLibrary.title)
|
.order_by(PlexLibrary.kind.desc(), collated(PlexLibrary.title, lang_for_user(user)))
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
@@ -237,7 +241,7 @@ def unified_library(
|
|||||||
if tsq is not None:
|
if tsq is not None:
|
||||||
order = [u.c.rank.desc()]
|
order = [u.c.rank.desc()]
|
||||||
else:
|
else:
|
||||||
col = sort_column(u, sort)
|
col = sort_column(u, sort, lang)
|
||||||
order = [(col.asc() if sort_dir == "asc" else col.desc()).nullslast()]
|
order = [(col.asc() if sort_dir == "asc" else col.desc()).nullslast()]
|
||||||
order.append(u.c.rk.desc())
|
order.append(u.c.rk.desc())
|
||||||
rows = db.execute(select(u).order_by(*order).offset(offset).limit(limit)).all()
|
rows = db.execute(select(u).order_by(*order).offset(offset).limit(limit)).all()
|
||||||
@@ -281,26 +285,32 @@ def unified_library(
|
|||||||
episodes = []
|
episodes = []
|
||||||
if tsq is not None and want_shows and field in ("all", "episode"):
|
if tsq is not None and want_shows and field in ("all", "episode"):
|
||||||
ep_st = aliased(PlexState)
|
ep_st = aliased(PlexState)
|
||||||
ep_vec = (
|
# In HU an episode is found by its Hungarian title too (E30: the overlay now fills it) — and
|
||||||
func.to_tsvector(TS_CONFIG, func.coalesce(PlexItem.title, ""))
|
# still by its English one, like `vector_match` does for the grid.
|
||||||
if field == "episode"
|
if field == "episode":
|
||||||
else PlexItem.search_vector
|
ep_vec = func.to_tsvector(TS_CONFIG, func.coalesce(PlexItem.title, ""))
|
||||||
)
|
ep_vec_hu = func.to_tsvector(TS_CONFIG, func.coalesce(PlexItem.title_hu, ""))
|
||||||
|
else:
|
||||||
|
ep_vec, ep_vec_hu = PlexItem.search_vector, PlexItem.search_vector_hu
|
||||||
|
hit = ep_vec.op("@@")(tsq)
|
||||||
|
if lang == "hu":
|
||||||
|
hit = or_(ep_vec_hu.op("@@")(tsq), hit)
|
||||||
|
ep_vec = ep_vec_hu
|
||||||
eps = db.execute(
|
eps = db.execute(
|
||||||
select(PlexItem, ep_st, PlexShow.title.label("show_title"))
|
select(PlexItem, ep_st, PlexShow)
|
||||||
.outerjoin(ep_st, and_(ep_st.item_id == PlexItem.id, ep_st.user_id == user.id))
|
.outerjoin(ep_st, and_(ep_st.item_id == PlexItem.id, ep_st.user_id == user.id))
|
||||||
.outerjoin(PlexShow, PlexShow.id == PlexItem.show_id)
|
.outerjoin(PlexShow, PlexShow.id == PlexItem.show_id)
|
||||||
.where(
|
.where(
|
||||||
PlexItem.kind == "episode",
|
PlexItem.kind == "episode",
|
||||||
PlexItem.library_id.in_(show_lib_ids),
|
PlexItem.library_id.in_(show_lib_ids),
|
||||||
ep_vec.op("@@")(tsq),
|
hit,
|
||||||
)
|
)
|
||||||
.order_by(func.ts_rank(ep_vec, tsq).desc())
|
.order_by(func.ts_rank(ep_vec, tsq).desc())
|
||||||
.limit(24)
|
.limit(24)
|
||||||
).all()
|
).all()
|
||||||
for e, s, show_title in eps:
|
for e, s, sh in eps:
|
||||||
card = episode_card(e, s)
|
card = episode_card(e, s, lang=lang)
|
||||||
card["show_title"] = show_title
|
card["show_title"] = display_title(sh, lang) if sh is not None else None
|
||||||
episodes.append(card)
|
episodes.append(card)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
@@ -348,6 +358,24 @@ def facets(
|
|||||||
|
|
||||||
uid = user.id
|
uid = user.id
|
||||||
|
|
||||||
|
# The shows that pass the watch-state scope, asked ONCE. The per-user episode rollup behind it is a
|
||||||
|
# full pass over the user's episodes (~10 ms on a prod-sized library), and each of the show groups
|
||||||
|
# (genres, ratings, studios, years, rating max) asked it again: half of a filtered facets call
|
||||||
|
# (E31 S4e). The same filter on the same rows, so the counts do not change
|
||||||
|
# (test_plex_facets_grid_parity).
|
||||||
|
shows_in_scope: list[int] | None = None
|
||||||
|
if show_ids and show in ("watched", "in_progress", "unwatched"):
|
||||||
|
agg = rollup.agg_subquery(db, uid).subquery()
|
||||||
|
in_scope = select(PlexShow.id).outerjoin(agg, agg.c.show_id == PlexShow.id)
|
||||||
|
in_scope = apply_show_state_filter(
|
||||||
|
in_scope,
|
||||||
|
func.coalesce(agg.c.total, 0),
|
||||||
|
func.coalesce(agg.c.watched, 0),
|
||||||
|
func.coalesce(agg.c.inprog, 0),
|
||||||
|
show,
|
||||||
|
)
|
||||||
|
shows_in_scope = list(db.scalars(in_scope.where(PlexShow.library_id.in_(show_ids))))
|
||||||
|
|
||||||
def wsq(query, model, ids, pp, kind_movie, with_dur):
|
def wsq(query, model, ids, pp, kind_movie, with_dur):
|
||||||
"""Apply the (self-excluded) metadata filters + library scope + the per-user watch-state to a
|
"""Apply the (self-excluded) metadata filters + library scope + the per-user watch-state to a
|
||||||
facet aggregation query, so the facets reflect exactly the visible result set."""
|
facet aggregation query, so the facets reflect exactly the visible result set."""
|
||||||
@@ -366,14 +394,8 @@ def facets(
|
|||||||
st = aliased(PlexState)
|
st = aliased(PlexState)
|
||||||
query = query.outerjoin(st, and_(st.item_id == model.id, st.user_id == uid))
|
query = query.outerjoin(st, and_(st.item_id == model.id, st.user_id == uid))
|
||||||
query = apply_movie_state_filter(query, st, show)
|
query = apply_movie_state_filter(query, st, show)
|
||||||
else:
|
elif shows_in_scope is not None:
|
||||||
if show in ("watched", "in_progress", "unwatched"):
|
query = query.where(model.id.in_(shows_in_scope))
|
||||||
agg = rollup.agg_subquery(db, uid).subquery()
|
|
||||||
query = query.outerjoin(agg, agg.c.show_id == model.id)
|
|
||||||
tot = func.coalesce(agg.c.total, 0)
|
|
||||||
wat = func.coalesce(agg.c.watched, 0)
|
|
||||||
inp = func.coalesce(agg.c.inprog, 0)
|
|
||||||
query = apply_show_state_filter(query, tot, wat, inp, show)
|
|
||||||
return query
|
return query
|
||||||
|
|
||||||
genre_counts: dict[str, int] = {}
|
genre_counts: dict[str, int] = {}
|
||||||
@@ -523,16 +545,19 @@ def facets(
|
|||||||
for c in split_csv(f.playable):
|
for c in split_csv(f.playable):
|
||||||
if c in PLAYBACK_CLASSES:
|
if c in PLAYBACK_CLASSES:
|
||||||
playable_counts.setdefault(c, 0)
|
playable_counts.setdefault(c, 0)
|
||||||
|
# Names in the reader's alphabet: the genres are Hungarian words for a Hungarian reader (E30 S1).
|
||||||
|
alpha = {w: i for i, w in enumerate(sorted_words(db, [*genre_counts, *studio_counts], lang))}
|
||||||
return {
|
return {
|
||||||
"genres": [
|
"genres": [
|
||||||
{"value": g, "count": c} for g, c in sorted(genre_counts.items(), key=lambda kv: kv[0])
|
{"value": g, "count": c}
|
||||||
|
for g, c in sorted(genre_counts.items(), key=lambda kv: alpha[kv[0]])
|
||||||
],
|
],
|
||||||
"content_ratings": [
|
"content_ratings": [
|
||||||
{"value": cr, "count": c} for cr, c in sorted(cr_counts.items(), key=lambda kv: -kv[1])
|
{"value": cr, "count": c} for cr, c in sorted(cr_counts.items(), key=lambda kv: -kv[1])
|
||||||
],
|
],
|
||||||
"studios": [
|
"studios": [
|
||||||
{"value": s, "count": c}
|
{"value": s, "count": c}
|
||||||
for s, c in sorted(studio_counts.items(), key=lambda kv: (-kv[1], kv[0]))
|
for s, c in sorted(studio_counts.items(), key=lambda kv: (-kv[1], alpha[kv[0]]))
|
||||||
],
|
],
|
||||||
# Fixed good→heavy order (not by count), so the class list reads as a severity scale.
|
# Fixed good→heavy order (not by count), so the class list reads as a severity scale.
|
||||||
"playable": [
|
"playable": [
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ from pydantic import BaseModel
|
|||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.mailtext import lang_for_user
|
||||||
|
from app.collation import collated, sorted_words
|
||||||
from app import logs
|
from app import logs
|
||||||
from app.auth import admin_user, current_user
|
from app.auth import admin_user, current_user
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
@@ -22,10 +24,10 @@ from app.models import (
|
|||||||
)
|
)
|
||||||
from app.plex import rollup
|
from app.plex import rollup
|
||||||
from app.plex import sync as plex_sync
|
from app.plex import sync as plex_sync
|
||||||
from app.plex.cards import movie_card, show_card
|
from app.plex.cards import display_title, movie_card, norm_lang, show_card
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
from app.plex.query import content_libs, runtime_by_key, top_art_keys
|
from app.plex.query import content_libs, runtime_by_key, title_col, top_art_keys
|
||||||
from app.utils import like_escape
|
from app.utils import like_escape
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
from app.routes._common import not_found
|
from app.routes._common import not_found
|
||||||
@@ -142,7 +144,7 @@ def collections(
|
|||||||
db.execute(
|
db.execute(
|
||||||
query.order_by(
|
query.order_by(
|
||||||
PlexCollection.child_count.desc().nullslast(),
|
PlexCollection.child_count.desc().nullslast(),
|
||||||
func.lower(PlexCollection.title),
|
collated(PlexCollection.title, lang_for_user(user)),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
@@ -208,16 +210,24 @@ MEMBER_SORT_KEYS: dict[str, Callable[[Member], Any]] = {
|
|||||||
DEFAULT_MEMBER_SORT = "year"
|
DEFAULT_MEMBER_SORT = "year"
|
||||||
|
|
||||||
|
|
||||||
def sort_members(members: list[Member], sort: str, sort_dir: str) -> list[Member]:
|
def sort_members(
|
||||||
|
members: list[Member], sort: str, sort_dir: str, alpha: dict[str, int] | None = None
|
||||||
|
) -> list[Member]:
|
||||||
"""Order a collection's members by one of `PLEX_SORT_KEYS`.
|
"""Order a collection's members by one of `PLEX_SORT_KEYS`.
|
||||||
|
|
||||||
Two properties worth stating because they are easy to get wrong with a plain `reverse=True`:
|
Two properties worth stating because they are easy to get wrong with a plain `reverse=True`:
|
||||||
members with no value for the key sink to the END in BOTH directions (matching the browse grid's
|
members with no value for the key sink to the END in BOTH directions (matching the browse grid's
|
||||||
`nullslast()` — a movie with no rating is not "the worst rated"), and ties always resolve
|
`nullslast()` — a movie with no rating is not "the worst rated"), and ties always resolve
|
||||||
title-ascending, so flipping the direction reverses the ranking rather than the alphabet inside
|
title-ascending, so flipping the direction reverses the ranking rather than the alphabet inside
|
||||||
it. An unknown key falls back to the page's default instead of returning an arbitrary order."""
|
it. An unknown key falls back to the page's default instead of returning an arbitrary order.
|
||||||
|
|
||||||
|
`alpha` ranks each title in the reader's alphabet (`collation.sorted_words`); without it, titles
|
||||||
|
compare as Python strings, which puts every accented initial after "z"."""
|
||||||
|
by_title = (lambda m: alpha[m.title]) if alpha is not None else (lambda m: m.title)
|
||||||
key = MEMBER_SORT_KEYS.get(sort, MEMBER_SORT_KEYS[DEFAULT_MEMBER_SORT])
|
key = MEMBER_SORT_KEYS.get(sort, MEMBER_SORT_KEYS[DEFAULT_MEMBER_SORT])
|
||||||
ordered = sorted(members, key=lambda m: m.title) # stable secondary, applied first
|
if key is MEMBER_SORT_KEYS["title"]:
|
||||||
|
key = by_title
|
||||||
|
ordered = sorted(members, key=by_title) # stable secondary, applied first
|
||||||
have = [m for m in ordered if key(m) is not None]
|
have = [m for m in ordered if key(m) is not None]
|
||||||
missing = [m for m in ordered if key(m) is None]
|
missing = [m for m in ordered if key(m) is None]
|
||||||
have.sort(key=key, reverse=sort_dir != "asc")
|
have.sort(key=key, reverse=sort_dir != "asc")
|
||||||
@@ -229,6 +239,7 @@ def collection_detail(
|
|||||||
rating_key: str,
|
rating_key: str,
|
||||||
sort: str = DEFAULT_MEMBER_SORT,
|
sort: str = DEFAULT_MEMBER_SORT,
|
||||||
sort_dir: str = "asc",
|
sort_dir: str = "asc",
|
||||||
|
lang: str = "en",
|
||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
@@ -237,6 +248,7 @@ def collection_detail(
|
|||||||
control means the same thing on both surfaces; the default is release year ascending, which is
|
control means the same thing on both surfaces; the default is release year ascending, which is
|
||||||
what this page did before it had a control. Read-only here; edits go through the admin mutation
|
what this page did before it had a control. Read-only here; edits go through the admin mutation
|
||||||
endpoints below."""
|
endpoints below."""
|
||||||
|
lang = norm_lang(lang)
|
||||||
col = by_rating_key(db, PlexCollection, rating_key)
|
col = by_rating_key(db, PlexCollection, rating_key)
|
||||||
if col is None:
|
if col is None:
|
||||||
raise not_found("collection")
|
raise not_found("collection")
|
||||||
@@ -244,7 +256,7 @@ def collection_detail(
|
|||||||
db.execute(
|
db.execute(
|
||||||
select(PlexItem)
|
select(PlexItem)
|
||||||
.where(PlexItem.kind == "movie", PlexItem.collection_keys.contains([rating_key]))
|
.where(PlexItem.kind == "movie", PlexItem.collection_keys.contains([rating_key]))
|
||||||
.order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title))
|
.order_by(PlexItem.year.asc().nullslast(), collated(title_col(PlexItem, lang), lang))
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
@@ -253,7 +265,7 @@ def collection_detail(
|
|||||||
db.execute(
|
db.execute(
|
||||||
select(PlexShow)
|
select(PlexShow)
|
||||||
.where(PlexShow.collection_keys.contains([rating_key]))
|
.where(PlexShow.collection_keys.contains([rating_key]))
|
||||||
.order_by(PlexShow.year.asc().nullslast(), func.lower(PlexShow.title))
|
.order_by(PlexShow.year.asc().nullslast(), collated(title_col(PlexShow, lang), lang))
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
@@ -285,7 +297,7 @@ def collection_detail(
|
|||||||
Member(
|
Member(
|
||||||
kind="movie",
|
kind="movie",
|
||||||
obj=m,
|
obj=m,
|
||||||
title=(m.title or "").lower(),
|
title=display_title(m, lang).lower(),
|
||||||
year=m.year,
|
year=m.year,
|
||||||
rating=m.rating,
|
rating=m.rating,
|
||||||
added=m.added_at,
|
added=m.added_at,
|
||||||
@@ -298,7 +310,7 @@ def collection_detail(
|
|||||||
Member(
|
Member(
|
||||||
kind="show",
|
kind="show",
|
||||||
obj=sh,
|
obj=sh,
|
||||||
title=(sh.title or "").lower(),
|
title=display_title(sh, lang).lower(),
|
||||||
year=sh.year,
|
year=sh.year,
|
||||||
rating=sh.rating,
|
rating=sh.rating,
|
||||||
added=sh.added_at,
|
added=sh.added_at,
|
||||||
@@ -308,11 +320,12 @@ def collection_detail(
|
|||||||
)
|
)
|
||||||
for sh in shows
|
for sh in shows
|
||||||
]
|
]
|
||||||
members = sort_members(members, sort, sort_dir)
|
alpha = {w: i for i, w in enumerate(sorted_words(db, [m.title for m in members], lang))}
|
||||||
|
members = sort_members(members, sort, sort_dir, alpha)
|
||||||
items = [
|
items = [
|
||||||
movie_card(m.obj, mstate.get(m.obj.id))
|
movie_card(m.obj, mstate.get(m.obj.id), lang=lang)
|
||||||
if m.kind == "movie"
|
if m.kind == "movie"
|
||||||
else show_card(m.obj, sstate.get(m.obj.id, "new"))
|
else show_card(m.obj, sstate.get(m.obj.id, "new"), lang=lang)
|
||||||
for m in members
|
for m in members
|
||||||
]
|
]
|
||||||
years = [m.year for m in members if m.year is not None]
|
years = [m.year for m in members if m.year is not None]
|
||||||
@@ -415,7 +428,7 @@ def create_collection(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except (PlexError, PlexNotConfigured) as e:
|
except (PlexError, PlexNotConfigured) as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise _write_or_pending_error(wrote, e)
|
raise _write_or_pending_error(wrote, e) from e
|
||||||
return _collection_card(col)
|
return _collection_card(col)
|
||||||
|
|
||||||
|
|
||||||
@@ -437,7 +450,7 @@ def collection_add_item(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except (PlexError, PlexNotConfigured) as e:
|
except (PlexError, PlexNotConfigured) as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise _write_or_pending_error(wrote, e)
|
raise _write_or_pending_error(wrote, e) from e
|
||||||
return _collection_card(col)
|
return _collection_card(col)
|
||||||
|
|
||||||
|
|
||||||
@@ -459,7 +472,7 @@ def collection_remove_item(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except (PlexError, PlexNotConfigured) as e:
|
except (PlexError, PlexNotConfigured) as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise _write_or_pending_error(wrote, e)
|
raise _write_or_pending_error(wrote, e) from e
|
||||||
return _collection_card(col)
|
return _collection_card(col)
|
||||||
|
|
||||||
|
|
||||||
@@ -489,7 +502,7 @@ def collection_rename(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except (PlexError, PlexNotConfigured) as e:
|
except (PlexError, PlexNotConfigured) as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise _write_or_pending_error(wrote, e)
|
raise _write_or_pending_error(wrote, e) from e
|
||||||
return _collection_card(col)
|
return _collection_card(col)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,14 +1,16 @@
|
|||||||
"""Plex show/item detail: the show drill-down + the per-item player payload (markers, cast, nav)."""
|
"""Plex show/item detail: the show drill-down + the per-item player payload (markers, cast, nav)."""
|
||||||
|
|
||||||
|
import re
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.collation import collated, sorted_words
|
||||||
from app import logs
|
from app import logs
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
@@ -24,6 +26,7 @@ from app.models import (
|
|||||||
from app.plex import detail as plex_detail
|
from app.plex import detail as plex_detail
|
||||||
from app.plex import people as plex_people
|
from app.plex import people as plex_people
|
||||||
from app.plex import rollup, subtitles
|
from app.plex import rollup, subtitles
|
||||||
|
from app.plex.query import title_col
|
||||||
from app.plex import tmdb as plex_tmdb
|
from app.plex import tmdb as plex_tmdb
|
||||||
from app.plex.cards import episode_card, movie_card, show_card
|
from app.plex.cards import episode_card, movie_card, show_card
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
@@ -107,7 +110,8 @@ def show_detail(
|
|||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""A show's seasons + episodes (the drill-down view) with per-user watch state."""
|
"""A show's seasons + episodes (the drill-down view) with per-user watch state."""
|
||||||
hu = _norm_lang(lang) == "hu"
|
lang = _norm_lang(lang)
|
||||||
|
hu = lang == "hu"
|
||||||
sh = by_rating_key(db, PlexShow, rating_key)
|
sh = by_rating_key(db, PlexShow, rating_key)
|
||||||
if sh is None:
|
if sh is None:
|
||||||
raise not_found("show")
|
raise not_found("show")
|
||||||
@@ -144,7 +148,7 @@ def show_detail(
|
|||||||
by_season: dict[int | None, list] = {}
|
by_season: dict[int | None, list] = {}
|
||||||
all_cards: list[dict] = [] # every episode, in season/episode order → show-level rollup
|
all_cards: list[dict] = [] # every episode, in season/episode order → show-level rollup
|
||||||
for e in eps:
|
for e in eps:
|
||||||
card = episode_card(e, states.get(e.id))
|
card = episode_card(e, states.get(e.id), lang=lang)
|
||||||
by_season.setdefault(e.season_id, []).append(card)
|
by_season.setdefault(e.season_id, []).append(card)
|
||||||
all_cards.append(card)
|
all_cards.append(card)
|
||||||
show_roll = rollup.rollup(all_cards)
|
show_roll = rollup.rollup(all_cards)
|
||||||
@@ -156,7 +160,7 @@ def show_detail(
|
|||||||
related: list[dict] = []
|
related: list[dict] = []
|
||||||
try:
|
try:
|
||||||
meta, related_raw = _show_enrichment(db, sh.rating_key)
|
meta, related_raw = _show_enrichment(db, sh.rating_key)
|
||||||
related = _relatedshow_cards(db, user.id, related_raw, exclude=sh.id)
|
related = _relatedshow_cards(db, user.id, related_raw, exclude=sh.id, lang=lang)
|
||||||
if sh.people_enriched_at is None and meta:
|
if sh.people_enriched_at is None and meta:
|
||||||
plex_people.enrich_row_people(db, sh, plex_people.cast_from_meta(meta))
|
plex_people.enrich_row_people(db, sh, plex_people.cast_from_meta(meta))
|
||||||
plex_detail.apply_rich_detail(sh, meta) # imdb/tagline/full_cast for future views
|
plex_detail.apply_rich_detail(sh, meta) # imdb/tagline/full_cast for future views
|
||||||
@@ -210,7 +214,7 @@ def show_detail(
|
|||||||
"episode_count": show_roll["episode_count"],
|
"episode_count": show_roll["episode_count"],
|
||||||
"collection_keys": sh.collection_keys or [],
|
"collection_keys": sh.collection_keys or [],
|
||||||
# TMDB enrichment (E5 S2), served from the row.
|
# TMDB enrichment (E5 S2), served from the row.
|
||||||
"tmdb": plex_tmdb.serve_block(db, sh),
|
"tmdb": plex_tmdb.serve_block(db, sh, lang=lang),
|
||||||
},
|
},
|
||||||
"seasons": [_season_block(se, by_season.get(se.id, [])) for se in seasons],
|
"seasons": [_season_block(se, by_season.get(se.id, [])) for se in seasons],
|
||||||
"related": related,
|
"related": related,
|
||||||
@@ -234,7 +238,9 @@ def _season_block(se: PlexSeason, cards: list[dict]) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _relatedshow_cards(db: Session, user_id: int, related: list[dict], exclude: int) -> list[dict]:
|
def _relatedshow_cards(
|
||||||
|
db: Session, user_id: int, related: list[dict], exclude: int, lang: str
|
||||||
|
) -> list[dict]:
|
||||||
"""Map Plex 'related' metadata to show cards for the shows we actually mirror (so they're openable),
|
"""Map Plex 'related' metadata to show cards for the shows we actually mirror (so they're openable),
|
||||||
excluding the current show. Order preserved; capped."""
|
excluding the current show. Order preserved; capped."""
|
||||||
rks = [str(m.get("ratingKey")) for m in related if m.get("ratingKey")]
|
rks = [str(m.get("ratingKey")) for m in related if m.get("ratingKey")]
|
||||||
@@ -254,13 +260,13 @@ def _relatedshow_cards(db: Session, user_id: int, related: list[dict], exclude:
|
|||||||
if sh is None or rk in seen:
|
if sh is None or rk in seen:
|
||||||
continue
|
continue
|
||||||
seen.add(rk)
|
seen.add(rk)
|
||||||
out.append(show_card(sh, statuses.get(sh.id, "new")))
|
out.append(show_card(sh, statuses.get(sh.id, "new"), lang=lang))
|
||||||
if len(out) >= 20:
|
if len(out) >= 20:
|
||||||
break
|
break
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
def _collection_strips(db: Session, it: PlexItem, user_id: int) -> list[dict]:
|
def _collection_strips(db: Session, it: PlexItem, user_id: int, lang: str) -> list[dict]:
|
||||||
"""The collections this movie belongs to, each with its member titles (playable cards) — the
|
"""The collections this movie belongs to, each with its member titles (playable cards) — the
|
||||||
'Avatar Collection' strip on the info page. Pure local lookup (no Plex call)."""
|
'Avatar Collection' strip on the info page. Pure local lookup (no Plex call)."""
|
||||||
keys = it.collection_keys or []
|
keys = it.collection_keys or []
|
||||||
@@ -273,7 +279,8 @@ def _collection_strips(db: Session, it: PlexItem, user_id: int) -> list[dict]:
|
|||||||
).scalars()
|
).scalars()
|
||||||
}
|
}
|
||||||
# Smallest (most specific — e.g. a franchise) first; the big auto-lists (IMDb/TMDb Popular) last.
|
# Smallest (most specific — e.g. a franchise) first; the big auto-lists (IMDb/TMDb Popular) last.
|
||||||
ordered = sorted((c for c in cols.values()), key=lambda c: (c.child_count or 0, c.title))
|
alpha = {w: i for i, w in enumerate(sorted_words(db, [c.title for c in cols.values()], lang))}
|
||||||
|
ordered = sorted(cols.values(), key=lambda c: (c.child_count or 0, alpha[c.title]))
|
||||||
out = []
|
out = []
|
||||||
for col in ordered:
|
for col in ordered:
|
||||||
rk = col.rating_key
|
rk = col.rating_key
|
||||||
@@ -285,7 +292,9 @@ def _collection_strips(db: Session, it: PlexItem, user_id: int) -> list[dict]:
|
|||||||
PlexItem.kind == "movie",
|
PlexItem.kind == "movie",
|
||||||
PlexItem.collection_keys.contains([rk]),
|
PlexItem.collection_keys.contains([rk]),
|
||||||
)
|
)
|
||||||
.order_by(PlexItem.year.asc().nullslast(), func.lower(PlexItem.title))
|
.order_by(
|
||||||
|
PlexItem.year.asc().nullslast(), collated(title_col(PlexItem, lang), lang)
|
||||||
|
)
|
||||||
.limit(60)
|
.limit(60)
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
@@ -305,7 +314,7 @@ def _collection_strips(db: Session, it: PlexItem, user_id: int) -> list[dict]:
|
|||||||
"id": rk,
|
"id": rk,
|
||||||
"title": col.title,
|
"title": col.title,
|
||||||
"source": _collection_source(col),
|
"source": _collection_source(col),
|
||||||
"items": [movie_card(m, states.get(m.id)) for m in members],
|
"items": [movie_card(m, states.get(m.id), lang=lang) for m in members],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
return out
|
return out
|
||||||
@@ -339,6 +348,26 @@ def _episode_nav(db: Session, it: PlexItem) -> tuple[str | None, str | None]:
|
|||||||
return (keys[i - 1] if i > 0 else None), (keys[i + 1] if i < len(keys) - 1 else None)
|
return (keys[i - 1] if i > 0 else None), (keys[i + 1] if i < len(keys) - 1 else None)
|
||||||
|
|
||||||
|
|
||||||
|
# The technical tail of Plex's track title — "English (AC3 5.1)" → "AC3 5.1".
|
||||||
|
_TRACK_TAIL = re.compile(r"\(([^()]+)\)\s*$")
|
||||||
|
|
||||||
|
|
||||||
|
def _track_naming(s: dict) -> dict:
|
||||||
|
"""What the player names a track from (E30). Plex's `displayTitle` is in the SERVER's language
|
||||||
|
("English (AC3 5.1)", "magyar Forced", "中文 (EAC3 Stereo)"), so it is sent only as the fallback
|
||||||
|
`label`; the client builds the name in the reader's language from the language tag, the
|
||||||
|
forced / SDH flags and the codec tail. No title and no language → no label (the client numbers it)."""
|
||||||
|
title = s.get("displayTitle") or s.get("language") or None
|
||||||
|
tail = _TRACK_TAIL.search(title) if title else None
|
||||||
|
return {
|
||||||
|
"label": title,
|
||||||
|
"lang_tag": s.get("languageTag") or None,
|
||||||
|
"forced": bool(s.get("forced")),
|
||||||
|
"hearing_impaired": bool(s.get("hearingImpaired")),
|
||||||
|
"detail": tail.group(1).strip() if tail else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/item/{rating_key}")
|
@router.get("/item/{rating_key}")
|
||||||
def item_detail(
|
def item_detail(
|
||||||
rating_key: str,
|
rating_key: str,
|
||||||
@@ -384,9 +413,7 @@ def item_detail(
|
|||||||
audio_streams.append(
|
audio_streams.append(
|
||||||
{
|
{
|
||||||
"ord": len(audio_streams),
|
"ord": len(audio_streams),
|
||||||
"label": s.get("displayTitle")
|
**_track_naming(s),
|
||||||
or s.get("language")
|
|
||||||
or f"Audio {len(audio_streams) + 1}",
|
|
||||||
"language": s.get("languageTag") or s.get("language"),
|
"language": s.get("languageTag") or s.get("language"),
|
||||||
"default": bool(s.get("selected") or s.get("default")),
|
"default": bool(s.get("selected") or s.get("default")),
|
||||||
}
|
}
|
||||||
@@ -396,9 +423,7 @@ def item_detail(
|
|||||||
subtitle_streams.append(
|
subtitle_streams.append(
|
||||||
{
|
{
|
||||||
"ord": len(subtitle_streams),
|
"ord": len(subtitle_streams),
|
||||||
"label": s.get("displayTitle")
|
**_track_naming(s),
|
||||||
or s.get("language")
|
|
||||||
or f"Subtitle {len(subtitle_streams) + 1}",
|
|
||||||
"language": s.get("languageTag") or s.get("language"),
|
"language": s.get("languageTag") or s.get("language"),
|
||||||
"codec": codec,
|
"codec": codec,
|
||||||
# Text subs (srt/ass/…) can be served as WebVTT tracks; image subs (PGS/VobSub)
|
# Text subs (srt/ass/…) can be served as WebVTT tracks; image subs (PGS/VobSub)
|
||||||
@@ -468,7 +493,7 @@ def item_detail(
|
|||||||
"genres": (it.genres_hu or it.genres if hu else it.genres) or [],
|
"genres": (it.genres_hu or it.genres if hu else it.genres) or [],
|
||||||
"directors": it.directors or [],
|
"directors": it.directors or [],
|
||||||
"studio": it.studio,
|
"studio": it.studio,
|
||||||
"tagline": it.tagline,
|
"tagline": (it.tagline_hu or it.tagline) if hu else it.tagline,
|
||||||
}
|
}
|
||||||
else:
|
else:
|
||||||
rich = plex_detail.rich_detail(meta)
|
rich = plex_detail.rich_detail(meta)
|
||||||
@@ -512,7 +537,7 @@ def item_detail(
|
|||||||
"episode_number": it.episode_number,
|
"episode_number": it.episode_number,
|
||||||
"prev_id": prev_id,
|
"prev_id": prev_id,
|
||||||
"next_id": next_id,
|
"next_id": next_id,
|
||||||
"collections": _collection_strips(db, it, user.id),
|
"collections": _collection_strips(db, it, user.id, lang),
|
||||||
# TMDB enrichment (E5 S2), served from the row — movies only (episodes aren't enriched).
|
# TMDB enrichment (E5 S2), served from the row — movies only (episodes aren't enriched).
|
||||||
"tmdb": plex_tmdb.serve_block(db, it) if it.kind == "movie" else None,
|
"tmdb": plex_tmdb.serve_block(db, it, lang=lang) if it.kind == "movie" else None,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,11 +18,12 @@ from fastapi import APIRouter, Depends, Query
|
|||||||
from sqlalchemy import and_, func, or_, select
|
from sqlalchemy import and_, func, or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.collation import sorted_words
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import get_db
|
from app.db import get_db
|
||||||
from app.models import PlexItem, PlexShow, PlexState, User
|
from app.models import PlexItem, PlexShow, PlexState, User
|
||||||
from app.plex import rollup
|
from app.plex import rollup
|
||||||
from app.plex.cards import leaf_card, movie_card, show_card
|
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||||
from app.plex.query import (
|
from app.plex.query import (
|
||||||
LibraryFilters,
|
LibraryFilters,
|
||||||
apply_meta_filters,
|
apply_meta_filters,
|
||||||
@@ -61,14 +62,14 @@ def _movie_meta(q, p: dict, lang: str, f: LibraryFilters):
|
|||||||
return q
|
return q
|
||||||
|
|
||||||
|
|
||||||
def _movie_cards(db: Session, uid: int, items: list[PlexItem]) -> list[dict]:
|
def _movie_cards(db: Session, uid: int, items: list[PlexItem], lang: str) -> list[dict]:
|
||||||
sts = rollup.item_state_map(db, uid, [it.id for it in items])
|
sts = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||||
return [movie_card(it, sts.get(it.id)) for it in items]
|
return [movie_card(it, sts.get(it.id), lang=lang) for it in items]
|
||||||
|
|
||||||
|
|
||||||
def _show_cards(db: Session, uid: int, shows: list[PlexShow]) -> list[dict]:
|
def _show_cards(db: Session, uid: int, shows: list[PlexShow], lang: str) -> list[dict]:
|
||||||
statuses = rollup.state_map(db, uid, [s.id for s in shows])
|
statuses = rollup.state_map(db, uid, [s.id for s in shows])
|
||||||
return [show_card(s, statuses.get(s.id, "new")) for s in shows]
|
return [show_card(s, statuses.get(s.id, "new"), lang=lang) for s in shows]
|
||||||
|
|
||||||
|
|
||||||
def _recently_added(
|
def _recently_added(
|
||||||
@@ -123,8 +124,14 @@ def _recently_added(
|
|||||||
# share one), so the merge doesn't depend on object identity staying alive.
|
# share one), so the merge doesn't depend on object identity staying alive.
|
||||||
top_movies = [o for _, kind, o in top if kind == "movie"]
|
top_movies = [o for _, kind, o in top if kind == "movie"]
|
||||||
top_shows = [o for _, kind, o in top if kind == "show"]
|
top_shows = [o for _, kind, o in top if kind == "show"]
|
||||||
mcards = {m.rating_key: c for m, c in zip(top_movies, _movie_cards(db, uid, top_movies))}
|
mcards = {
|
||||||
scards = {s.rating_key: c for s, c in zip(top_shows, _show_cards(db, uid, top_shows))}
|
m.rating_key: c
|
||||||
|
for m, c in zip(top_movies, _movie_cards(db, uid, top_movies, lang), strict=True)
|
||||||
|
}
|
||||||
|
scards = {
|
||||||
|
s.rating_key: c
|
||||||
|
for s, c in zip(top_shows, _show_cards(db, uid, top_shows, lang), strict=True)
|
||||||
|
}
|
||||||
return [mcards[o.rating_key] if kind == "movie" else scards[o.rating_key] for _, kind, o in top]
|
return [mcards[o.rating_key] if kind == "movie" else scards[o.rating_key] for _, kind, o in top]
|
||||||
|
|
||||||
|
|
||||||
@@ -156,10 +163,10 @@ def _recently_watched(
|
|||||||
.order_by(PlexState.watched_at.desc().nullslast())
|
.order_by(PlexState.watched_at.desc().nullslast())
|
||||||
.limit(RAIL_LIMIT)
|
.limit(RAIL_LIMIT)
|
||||||
).all():
|
).all():
|
||||||
rows.append((st.watched_at, leaf_card(it, st)))
|
rows.append((st.watched_at, leaf_card(it, st, lang=lang)))
|
||||||
if show_ids:
|
if show_ids:
|
||||||
eq = (
|
eq = (
|
||||||
select(PlexItem, PlexState, PlexShow.title, PlexShow.rating_key)
|
select(PlexItem, PlexState, PlexShow)
|
||||||
.join(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
|
.join(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
|
||||||
.join(PlexShow, PlexShow.id == PlexItem.show_id)
|
.join(PlexShow, PlexShow.id == PlexItem.show_id)
|
||||||
.where(
|
.where(
|
||||||
@@ -170,8 +177,11 @@ def _recently_watched(
|
|||||||
.order_by(PlexState.watched_at.desc().nullslast())
|
.order_by(PlexState.watched_at.desc().nullslast())
|
||||||
.limit(RAIL_LIMIT)
|
.limit(RAIL_LIMIT)
|
||||||
)
|
)
|
||||||
for ep, st, sh_title, sh_rk in db.execute(eq).all():
|
for ep, st, sh in db.execute(eq).all():
|
||||||
rows.append((st.watched_at, leaf_card(ep, st, show_title=sh_title, show_rk=sh_rk)))
|
card = leaf_card(
|
||||||
|
ep, st, show_title=display_title(sh, lang), show_rk=sh.rating_key, lang=lang
|
||||||
|
)
|
||||||
|
rows.append((st.watched_at, card))
|
||||||
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
||||||
return [card for _, card in rows[:RAIL_LIMIT]]
|
return [card for _, card in rows[:RAIL_LIMIT]]
|
||||||
|
|
||||||
@@ -214,7 +224,7 @@ def _continue_watching(
|
|||||||
.limit(RAIL_LIMIT)
|
.limit(RAIL_LIMIT)
|
||||||
).all()
|
).all()
|
||||||
for it, st in mrows:
|
for it, st in mrows:
|
||||||
rows.append((st.progress_updated_at, movie_card(it, st)))
|
rows.append((st.progress_updated_at, movie_card(it, st, lang=lang)))
|
||||||
|
|
||||||
if show_ids:
|
if show_ids:
|
||||||
# Reuse the shared per-show episode rollup (total/watched/inprog) rather than re-inlining it, plus
|
# Reuse the shared per-show episode rollup (total/watched/inprog) rather than re-inlining it, plus
|
||||||
@@ -234,7 +244,7 @@ def _continue_watching(
|
|||||||
.limit(RAIL_LIMIT)
|
.limit(RAIL_LIMIT)
|
||||||
).all()
|
).all()
|
||||||
for sh, resumed_at in srows:
|
for sh, resumed_at in srows:
|
||||||
rows.append((resumed_at, show_card(sh, "in_progress")))
|
rows.append((resumed_at, show_card(sh, "in_progress", lang=lang)))
|
||||||
|
|
||||||
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
rows.sort(key=lambda r: r[0] or rollup.NO_TS, reverse=True)
|
||||||
return [card for _, card in rows[:RAIL_LIMIT]]
|
return [card for _, card in rows[:RAIL_LIMIT]]
|
||||||
@@ -261,7 +271,8 @@ def _genre_tiles(
|
|||||||
expanded = base.subquery()
|
expanded = base.subquery()
|
||||||
for g, c in db.execute(select(expanded.c.g, func.count()).group_by(expanded.c.g)).all():
|
for g, c in db.execute(select(expanded.c.g, func.count()).group_by(expanded.c.g)).all():
|
||||||
counts[g] = counts.get(g, 0) + c
|
counts[g] = counts.get(g, 0) + c
|
||||||
top = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0]))[:GENRE_TILES]
|
alpha = {w: i for i, w in enumerate(sorted_words(db, list(counts), lang))}
|
||||||
|
top = sorted(counts.items(), key=lambda kv: (-kv[1], alpha[kv[0]]))[:GENRE_TILES]
|
||||||
# A 2×2 poster collage per tile: the top movies in that genre (color hash is the client fallback).
|
# A 2×2 poster collage per tile: the top movies in that genre (color hash is the client fallback).
|
||||||
art = top_art_keys(
|
art = top_art_keys(
|
||||||
db,
|
db,
|
||||||
@@ -310,7 +321,7 @@ def _because_you_watched(
|
|||||||
)
|
)
|
||||||
if anchor is None:
|
if anchor is None:
|
||||||
return None
|
return None
|
||||||
resolved = resolve_recs(db, anchor.recommendations)
|
resolved = resolve_recs(db, anchor.recommendations, lang=lang)
|
||||||
# Preserve TMDB's recommendation order, keep only in-library hits, and drop the anchor itself.
|
# Preserve TMDB's recommendation order, keep only in-library hits, and drop the anchor itself.
|
||||||
rks = [
|
rks = [
|
||||||
r["rating_key"]
|
r["rating_key"]
|
||||||
@@ -346,9 +357,13 @@ def _because_you_watched(
|
|||||||
ordered_shows = [shows[rk] for rk in rks if rk in shows]
|
ordered_shows = [shows[rk] for rk in rks if rk in shows]
|
||||||
# rating_key is globally unique in Plex, so it keys the serialized cards back safely (no id()).
|
# rating_key is globally unique in Plex, so it keys the serialized cards back safely (no id()).
|
||||||
mcards = {
|
mcards = {
|
||||||
m.rating_key: c for m, c in zip(ordered_movies, _movie_cards(db, uid, ordered_movies))
|
m.rating_key: c
|
||||||
|
for m, c in zip(ordered_movies, _movie_cards(db, uid, ordered_movies, lang), strict=True)
|
||||||
|
}
|
||||||
|
scards = {
|
||||||
|
s.rating_key: c
|
||||||
|
for s, c in zip(ordered_shows, _show_cards(db, uid, ordered_shows, lang), strict=True)
|
||||||
}
|
}
|
||||||
scards = {s.rating_key: c for s, c in zip(ordered_shows, _show_cards(db, uid, ordered_shows))}
|
|
||||||
items: list[dict] = []
|
items: list[dict] = []
|
||||||
for rk in rks:
|
for rk in rks:
|
||||||
if rk in mcards:
|
if rk in mcards:
|
||||||
@@ -357,7 +372,10 @@ def _because_you_watched(
|
|||||||
items.append(scards[rk])
|
items.append(scards[rk])
|
||||||
if not items:
|
if not items:
|
||||||
return None
|
return None
|
||||||
return {"anchor": {"id": anchor.rating_key, "title": anchor.title}, "items": items}
|
return {
|
||||||
|
"anchor": {"id": anchor.rating_key, "title": display_title(anchor, lang)},
|
||||||
|
"items": items,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@router.get("/discover")
|
@router.get("/discover")
|
||||||
@@ -456,8 +474,8 @@ def surprise(
|
|||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
mcards = _movie_cards(db, user.id, movies)
|
mcards = _movie_cards(db, user.id, movies, lang)
|
||||||
scards = _show_cards(db, user.id, shows)
|
scards = _show_cards(db, user.id, shows, lang)
|
||||||
# Interleave movie/show cards, then trim to `limit`.
|
# Interleave movie/show cards, then trim to `limit`.
|
||||||
items: list[dict] = []
|
items: list[dict] = []
|
||||||
for i in range(max(len(mcards), len(scards))):
|
for i in range(max(len(mcards), len(scards))):
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ from fastapi import APIRouter, BackgroundTasks, Depends
|
|||||||
from sqlalchemy import func, or_, select
|
from sqlalchemy import func, or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs
|
from app.collation import collated
|
||||||
|
from app import logs, sysconfig
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import SessionLocal, get_db
|
from app.db import SessionLocal, get_db
|
||||||
from app.models import (
|
from app.models import (
|
||||||
@@ -18,22 +19,28 @@ from app.models import (
|
|||||||
)
|
)
|
||||||
from app.plex import rollup
|
from app.plex import rollup
|
||||||
from app.plex import tmdb as plex_tmdb
|
from app.plex import tmdb as plex_tmdb
|
||||||
from app.plex.cards import movie_card, show_card
|
from app.plex.cards import movie_card, norm_lang, show_card
|
||||||
from app.plex.query import content_libs, split_csv
|
from app.plex.query import content_libs, split_csv, title_col
|
||||||
|
|
||||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# How many "Known for" titles the person page shows (the stored pool is deeper; see plex.tmdb).
|
||||||
|
_KNOWN_FOR_SHOWN = 15
|
||||||
|
|
||||||
|
|
||||||
def _resolve_known_for(
|
def _resolve_known_for(
|
||||||
db: Session,
|
db: Session,
|
||||||
known_for: list | None,
|
known_for: list | None,
|
||||||
uid: int,
|
uid: int,
|
||||||
movie_lib_ids: list[int],
|
movie_lib_ids: list[int],
|
||||||
show_lib_ids: list[int],
|
show_lib_ids: list[int],
|
||||||
|
lang: str,
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""The person's TMDB "Known for" credits (top by popularity) that are IN our library, as cards, in
|
"""The person's TMDB "Known for" credits (most-voted first) that are IN our library, as cards, in
|
||||||
popularity order. Matched by the canonical guid `tmdb:<id>`; out-of-library credits are dropped."""
|
that order, at most _KNOWN_FOR_SHOWN. Matched by the canonical guid `tmdb:<id>`; out-of-library
|
||||||
|
credits are dropped."""
|
||||||
if not known_for:
|
if not known_for:
|
||||||
return []
|
return []
|
||||||
# Match each credit to a library row by guid, bucketed by media_type (movie ids and tv ids collide
|
# Match each credit to a library row by guid, bucketed by media_type (movie ids and tv ids collide
|
||||||
@@ -50,10 +57,10 @@ def _resolve_known_for(
|
|||||||
# A credit resolves against its OWN media_type's bucket only.
|
# A credit resolves against its OWN media_type's bucket only.
|
||||||
if k.get("media_type") == "tv":
|
if k.get("media_type") == "tv":
|
||||||
if g in shows:
|
if g in shows:
|
||||||
out.append(show_card(shows[g], show_statuses.get(shows[g].id, "new")))
|
out.append(show_card(shows[g], show_statuses.get(shows[g].id, "new"), lang=lang))
|
||||||
elif g in movies:
|
elif g in movies:
|
||||||
out.append(movie_card(movies[g], m_states.get(movies[g].id)))
|
out.append(movie_card(movies[g], m_states.get(movies[g].id), lang=lang))
|
||||||
return out
|
return out[:_KNOWN_FOR_SHOWN]
|
||||||
|
|
||||||
|
|
||||||
@router.get("/people/cards")
|
@router.get("/people/cards")
|
||||||
@@ -159,6 +166,7 @@ def _enrich_person_bg(person_id: int) -> None:
|
|||||||
def person_detail(
|
def person_detail(
|
||||||
name: str,
|
name: str,
|
||||||
background_tasks: BackgroundTasks,
|
background_tasks: BackgroundTasks,
|
||||||
|
lang: str = "en",
|
||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
@@ -171,18 +179,24 @@ def person_detail(
|
|||||||
E5 S6: the first time a page is opened, the person's TMDB bio + "Known for" credits are lazily
|
E5 S6: the first time a page is opened, the person's TMDB bio + "Known for" credits are lazily
|
||||||
fetched (from the S2-linked `tmdb_person_id`) and persisted, then served from the row on every open.
|
fetched (from the S2-linked `tmdb_person_id`) and persisted, then served from the row on every open.
|
||||||
The fetch runs OFF the request path (a background task); the first open serves whatever's in the row
|
The fetch runs OFF the request path (a background task); the first open serves whatever's in the row
|
||||||
(bio possibly still null) and the bio lands from the row by the next open.
|
with `enriching` set, and the page asks again until the bio lands. E30 S6: the people_enrich job
|
||||||
|
fetches everyone ahead of time, so this lazy path is only for a person the job hasn't reached yet.
|
||||||
"""
|
"""
|
||||||
|
lang = norm_lang(lang)
|
||||||
person = db.execute(select(PlexPerson).where(PlexPerson.name == name)).scalars().first()
|
person = db.execute(select(PlexPerson).where(PlexPerson.name == name)).scalars().first()
|
||||||
# Lazy one-time bio enrich, scheduled AFTER the response so the TMDB call is never on the serve path.
|
# Lazy one-time bio enrich, scheduled AFTER the response so the TMDB call is never on the serve path.
|
||||||
# Gated by a per-person cooldown so concurrent/rapid opens (or repeated opens while TMDB is degraded)
|
# Gated by a per-person cooldown so concurrent/rapid opens (or repeated opens while TMDB is degraded)
|
||||||
# can't pile background tasks onto the shared threadpool.
|
# can't pile background tasks onto the shared threadpool.
|
||||||
if (
|
# `enriching` tells the page the bio is on its way (E30 S6): it asks again shortly instead of showing
|
||||||
|
# an empty page until the next open. True while a fetch is pending, including one an earlier open
|
||||||
|
# already reserved.
|
||||||
|
enriching = (
|
||||||
person is not None
|
person is not None
|
||||||
and person.tmdb_person_id
|
and bool(person.tmdb_person_id)
|
||||||
and person.tmdb_enriched_at is None
|
and person.tmdb_enriched_at is None
|
||||||
and _should_schedule_enrich(person.id)
|
and bool(sysconfig.get_str(db, "tmdb_api_key"))
|
||||||
):
|
)
|
||||||
|
if enriching and person is not None and _should_schedule_enrich(person.id):
|
||||||
background_tasks.add_task(_enrich_person_bg, person.id)
|
background_tasks.add_task(_enrich_person_bg, person.id)
|
||||||
movie_lib_ids, show_lib_ids = content_libs(db)
|
movie_lib_ids, show_lib_ids = content_libs(db)
|
||||||
|
|
||||||
@@ -199,14 +213,16 @@ def person_detail(
|
|||||||
PlexItem.directors.contains([name]),
|
PlexItem.directors.contains([name]),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.order_by(PlexItem.year.desc().nullslast(), func.lower(PlexItem.title))
|
.order_by(
|
||||||
|
PlexItem.year.desc().nullslast(), collated(title_col(PlexItem, lang), lang)
|
||||||
|
)
|
||||||
.limit(_PERSON_SECTION_MAX)
|
.limit(_PERSON_SECTION_MAX)
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
states = rollup.item_state_map(db, user.id, [it.id for it in items])
|
states = rollup.item_state_map(db, user.id, [it.id for it in items])
|
||||||
movies = [movie_card(it, states.get(it.id)) for it in items]
|
movies = [movie_card(it, states.get(it.id), lang=lang) for it in items]
|
||||||
|
|
||||||
shows: list[dict] = []
|
shows: list[dict] = []
|
||||||
if show_lib_ids:
|
if show_lib_ids:
|
||||||
@@ -220,28 +236,34 @@ def person_detail(
|
|||||||
PlexShow.directors.contains([name]),
|
PlexShow.directors.contains([name]),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
.order_by(PlexShow.year.desc().nullslast(), func.lower(PlexShow.title))
|
.order_by(
|
||||||
|
PlexShow.year.desc().nullslast(), collated(title_col(PlexShow, lang), lang)
|
||||||
|
)
|
||||||
.limit(_PERSON_SECTION_MAX)
|
.limit(_PERSON_SECTION_MAX)
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
statuses = rollup.state_map(db, user.id, [s.id for s in show_rows])
|
statuses = rollup.state_map(db, user.id, [s.id for s in show_rows])
|
||||||
shows = [show_card(sh, statuses.get(sh.id, "new")) for sh in show_rows]
|
shows = [show_card(sh, statuses.get(sh.id, "new"), lang=lang) for sh in show_rows]
|
||||||
|
|
||||||
known_for = _resolve_known_for(
|
known_for = _resolve_known_for(
|
||||||
db, person.known_for if person else None, user.id, movie_lib_ids, show_lib_ids
|
db, person.known_for if person else None, user.id, movie_lib_ids, show_lib_ids, lang
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"name": name,
|
"name": name,
|
||||||
"photo": person.photo_thumb if person else None,
|
"photo": person.photo_thumb if person else None,
|
||||||
"biography": person.biography if person else None,
|
# The Hungarian biography for a Hungarian reader, the instance-language one when TMDB has none.
|
||||||
|
"biography": ((person.biography_hu if lang == "hu" else None) or person.biography)
|
||||||
|
if person
|
||||||
|
else None,
|
||||||
"birthday": person.birthday if person else None,
|
"birthday": person.birthday if person else None,
|
||||||
"deathday": person.deathday if person else None,
|
"deathday": person.deathday if person else None,
|
||||||
"place_of_birth": person.place_of_birth if person else None,
|
"place_of_birth": person.place_of_birth if person else None,
|
||||||
"known_for_department": person.known_for_department if person else None,
|
"known_for_department": person.known_for_department if person else None,
|
||||||
"also_known_as": (person.also_known_as if person else None) or [],
|
"also_known_as": (person.also_known_as if person else None) or [],
|
||||||
"known_for": known_for,
|
"known_for": known_for,
|
||||||
|
"enriching": enriching,
|
||||||
"movies": movies,
|
"movies": movies,
|
||||||
"shows": shows,
|
"shows": shows,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
"""Plex playback: image/subtitle proxies, HLS streaming, and watch-state writes."""
|
"""Plex playback: image/subtitle proxies, HLS streaming, and watch-state writes."""
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import threading
|
import threading
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
@@ -66,7 +65,7 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response
|
|||||||
brief session only to resolve the image key + read the Plex config, then RELEASES it before the
|
brief session only to resolve the image key + read the Plex config, then RELEASES it before the
|
||||||
fetch. Auth is the signed-session check (no DB user-load)."""
|
fetch. Auth is the signed-session check (no DB user-load)."""
|
||||||
_require_session(request)
|
_require_session(request)
|
||||||
v = "art" if variant == "art" else "thumb"
|
v = variant if variant in imgcache.SIZES else "thumb"
|
||||||
w, h = imgcache.SIZES[v]
|
w, h = imgcache.SIZES[v]
|
||||||
# Size is part of the cache key: Plex resizes server-side (see PlexClient.image_bytes), so we cache
|
# Size is part of the cache key: Plex resizes server-side (see PlexClient.image_bytes), so we cache
|
||||||
# the small variant. The width tag also means changing a target size just re-caches, never serves a
|
# the small variant. The width tag also means changing a target size just re-caches, never serves a
|
||||||
@@ -78,7 +77,7 @@ def image(rating_key: str, request: Request, variant: str = "thumb") -> Response
|
|||||||
# Cold miss: short-lived session for the key + Plex client config, released before the slow fetch.
|
# Cold miss: short-lived session for the key + Plex client config, released before the slow fetch.
|
||||||
try:
|
try:
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
key = imgcache.key_for(db, str(rating_key), v)
|
key = imgcache.key_for(db, str(rating_key), imgcache.picture(v))
|
||||||
if not key:
|
if not key:
|
||||||
raise not_found("image")
|
raise not_found("image")
|
||||||
plex = PlexClient(db) # reads base URL + token now; the fetch below needs no DB
|
plex = PlexClient(db) # reads base URL + token now; the fetch below needs no DB
|
||||||
@@ -214,24 +213,31 @@ def _resume_push_superseded(user_id: int, item_id: int, intent: datetime) -> boo
|
|||||||
|
|
||||||
@router.get("/person-image")
|
@router.get("/person-image")
|
||||||
def person_image(u: str, request: Request) -> Response:
|
def person_image(u: str, request: Request) -> Response:
|
||||||
"""Proxy a cast/crew photo from Plex's public metadata CDN (host-whitelisted; no token needed).
|
"""Proxy a cast/crew photo from Plex's public metadata CDN (only a headshot-shaped URL on its host;
|
||||||
Signed-session auth (no DB) so it, like /image, never holds a pool connection during the fetch."""
|
no token needed), resized through Plex and cached (plex.people.cache_portrait; the people_enrich job
|
||||||
|
fills the same cache ahead of time). Signed-session auth; like /image, the DB session is released
|
||||||
|
before the fetch."""
|
||||||
_require_session(request)
|
_require_session(request)
|
||||||
if not u.startswith(plex_people.PERSON_IMG_HOST):
|
if not plex_people.is_portrait_url(u):
|
||||||
raise refused(400, "image_host")
|
raise refused(400, "image_host")
|
||||||
cache_key = "person_" + hashlib.sha1(u.encode()).hexdigest()
|
hit = plex_people.portrait_cached(u)
|
||||||
hit = imgcache.read(cache_key)
|
|
||||||
if hit:
|
if hit:
|
||||||
return imgcache.response(*hit)
|
return imgcache.response(*hit)
|
||||||
try:
|
try:
|
||||||
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
with SessionLocal() as db:
|
||||||
r = c.get(u)
|
plex: PlexClient | None = PlexClient(db)
|
||||||
r.raise_for_status()
|
# silent-ok: Plex off is a setting; without its resize the original comes straight from the CDN.
|
||||||
except httpx.HTTPError:
|
except PlexNotConfigured:
|
||||||
raise refused(502, "image_fetch_failed")
|
plex = None
|
||||||
ct = r.headers.get("content-type", "image/jpeg")
|
try:
|
||||||
imgcache.write(cache_key, r.content, ct)
|
if plex is None:
|
||||||
return imgcache.response(r.content, ct)
|
data, ct = plex_people.cache_portrait(None, u)
|
||||||
|
else:
|
||||||
|
with plex:
|
||||||
|
data, ct = plex_people.cache_portrait(plex, u)
|
||||||
|
except (PlexError, httpx.HTTPError):
|
||||||
|
raise refused(502, "image_fetch_failed") from None
|
||||||
|
return imgcache.response(data, ct)
|
||||||
|
|
||||||
|
|
||||||
def _push_resume_guarded(
|
def _push_resume_guarded(
|
||||||
|
|||||||
@@ -3,8 +3,10 @@
|
|||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends
|
||||||
from pydantic import BaseModel
|
from typing import Annotated
|
||||||
from sqlalchemy import delete, func, select, update
|
|
||||||
|
from pydantic import BaseModel, Field, model_validator
|
||||||
|
from sqlalchemy import delete, func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
@@ -17,7 +19,7 @@ from app.models import (
|
|||||||
PlexState,
|
PlexState,
|
||||||
User,
|
User,
|
||||||
)
|
)
|
||||||
from app.plex.cards import leaf_card
|
from app.plex.cards import display_title, leaf_card, norm_lang
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
from app.routes._common import owned_or_404, not_found
|
from app.routes._common import owned_or_404, not_found
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
@@ -135,8 +137,12 @@ def create_playlist(
|
|||||||
|
|
||||||
@router.get("/playlists/{pid}")
|
@router.get("/playlists/{pid}")
|
||||||
def playlist_detail(
|
def playlist_detail(
|
||||||
pid: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
pid: int,
|
||||||
|
lang: str = "en",
|
||||||
|
user: User = Depends(current_user),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
lang = norm_lang(lang)
|
||||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
||||||
rows = db.execute(_playlist_item_query(pid)).all()
|
rows = db.execute(_playlist_item_query(pid)).all()
|
||||||
items = [it for (_pi, it) in rows]
|
items = [it for (_pi, it) in rows]
|
||||||
@@ -161,8 +167,9 @@ def playlist_detail(
|
|||||||
leaf_card(
|
leaf_card(
|
||||||
it,
|
it,
|
||||||
states.get(it.id),
|
states.get(it.id),
|
||||||
shows[it.show_id].title if it.show_id in shows else None,
|
display_title(shows[it.show_id], lang) if it.show_id in shows else None,
|
||||||
shows[it.show_id].rating_key if it.show_id in shows else None,
|
shows[it.show_id].rating_key if it.show_id in shows else None,
|
||||||
|
lang=lang,
|
||||||
)
|
)
|
||||||
for it in items
|
for it in items
|
||||||
]
|
]
|
||||||
@@ -217,7 +224,8 @@ def playlist_add_item(
|
|||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
# Locked (like every add / reorder here): two adds otherwise read the same max position.
|
||||||
|
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||||
it = by_rating_key(db, PlexItem, body.item_rating_key)
|
it = by_rating_key(db, PlexItem, body.item_rating_key)
|
||||||
if it is None:
|
if it is None:
|
||||||
raise not_found("item")
|
raise not_found("item")
|
||||||
@@ -249,7 +257,9 @@ def playlist_remove_item(
|
|||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
# Locked like the adds and the reorder: the list row first, then its items — a remove that
|
||||||
|
# locked an item row before a reorder locked the list deadlocked against it.
|
||||||
|
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||||
it = by_rating_key(db, PlexItem, item_rating_key)
|
it = by_rating_key(db, PlexItem, item_rating_key)
|
||||||
if it is not None:
|
if it is not None:
|
||||||
db.execute(
|
db.execute(
|
||||||
@@ -285,16 +295,75 @@ def _items_by_rks(db: Session, rks: list) -> list[PlexItem]:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
class PlexItemAddBulkIn(PlexItemKeysIn):
|
||||||
|
# Where each key goes back (E28 S3e2, the list page's "Undo" of a removal): an index over the
|
||||||
|
# ORDERED rows, one per key. The two-call shape (append, then send the whole order) leaves the rows
|
||||||
|
# at the end whenever the second call fails, and the order it sends would overwrite anything
|
||||||
|
# another tab changed in between.
|
||||||
|
positions: list[Annotated[int, Field(ge=0)]] | None = None
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def _one_position_per_key(self):
|
||||||
|
if self.positions is not None and len(self.positions) != len(self.item_rating_keys):
|
||||||
|
raise ValueError("positions must pair one-to-one with item_rating_keys")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_at(db: Session, pid: int, placed: list[tuple[int, PlexItem]]) -> int:
|
||||||
|
"""Insert new rows at their indices (ascending, so each index is where the row stood when the
|
||||||
|
rows before it were back), renumber the whole list 0..n-1, and say how many were added. An item
|
||||||
|
already in the list does not move: the add stays idempotent."""
|
||||||
|
rows = list(
|
||||||
|
db.execute(
|
||||||
|
select(PlexPlaylistItem)
|
||||||
|
.where(PlexPlaylistItem.playlist_id == pid)
|
||||||
|
.order_by(PlexPlaylistItem.position, PlexPlaylistItem.id)
|
||||||
|
).scalars()
|
||||||
|
)
|
||||||
|
present = {r.item_id for r in rows}
|
||||||
|
added = 0
|
||||||
|
for at, it in sorted(placed, key=lambda p: p[0]):
|
||||||
|
if it.id in present:
|
||||||
|
continue
|
||||||
|
row = PlexPlaylistItem(playlist_id=pid, item_id=it.id, position=0)
|
||||||
|
db.add(row)
|
||||||
|
rows.insert(min(at, len(rows)), row)
|
||||||
|
present.add(it.id)
|
||||||
|
added += 1
|
||||||
|
for i, r in enumerate(rows):
|
||||||
|
r.position = i
|
||||||
|
return added
|
||||||
|
|
||||||
|
|
||||||
@router.post("/playlists/{pid}/items/bulk")
|
@router.post("/playlists/{pid}/items/bulk")
|
||||||
def playlist_add_bulk(
|
def playlist_add_bulk(
|
||||||
pid: int,
|
pid: int,
|
||||||
body: PlexItemKeysIn,
|
body: PlexItemAddBulkIn,
|
||||||
user: User = Depends(current_user),
|
user: User = Depends(current_user),
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Append many items at once (a whole season/show). Already-present items are skipped; the input
|
"""Append many items at once (a whole season/show). Already-present items are skipped; the input
|
||||||
order is preserved for the newly-added ones. Returns how many were added + the new total."""
|
order is preserved for the newly-added ones. With `positions`, each key is put back at its index
|
||||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
instead of appended. Returns how many were added + the new total."""
|
||||||
|
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||||
|
if body.positions is not None:
|
||||||
|
by_rk = {it.rating_key: it for it in _items_by_rks(db, body.item_rating_keys)}
|
||||||
|
placed = [
|
||||||
|
(at, by_rk[str(rk)])
|
||||||
|
for rk, at in zip(body.item_rating_keys, body.positions, strict=True)
|
||||||
|
if str(rk) in by_rk
|
||||||
|
]
|
||||||
|
added = _insert_at(db, pid, placed)
|
||||||
|
if added:
|
||||||
|
pl.updated_at = datetime.now(timezone.utc)
|
||||||
|
db.commit()
|
||||||
|
total = (
|
||||||
|
db.scalar(
|
||||||
|
select(func.count(PlexPlaylistItem.id)).where(PlexPlaylistItem.playlist_id == pid)
|
||||||
|
)
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
return {"ok": True, "added": added, "item_count": total}
|
||||||
items = _items_by_rks(db, body.item_rating_keys)
|
items = _items_by_rks(db, body.item_rating_keys)
|
||||||
existing = set(
|
existing = set(
|
||||||
db.execute(
|
db.execute(
|
||||||
@@ -337,7 +406,7 @@ def playlist_remove_bulk(
|
|||||||
) -> dict:
|
) -> dict:
|
||||||
"""Remove many items at once (a whole season/show, or a multi-select). Returns how many were
|
"""Remove many items at once (a whole season/show, or a multi-select). Returns how many were
|
||||||
removed + the new total."""
|
removed + the new total."""
|
||||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||||
ids = [it.id for it in _items_by_rks(db, body.item_rating_keys)]
|
ids = [it.id for it in _items_by_rks(db, body.item_rating_keys)]
|
||||||
removed = 0
|
removed = 0
|
||||||
if ids:
|
if ids:
|
||||||
@@ -367,7 +436,7 @@ def reorder_playlist(
|
|||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
"""Set the playlist order from an explicit list of item rating_keys (0-based positions)."""
|
"""Set the playlist order from an explicit list of item rating_keys (0-based positions)."""
|
||||||
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist")
|
pl = owned_or_404(db, PlexPlaylist, pid, user, what="playlist", lock=True)
|
||||||
order = body.item_rating_keys
|
order = body.item_rating_keys
|
||||||
id_by_rk = {
|
id_by_rk = {
|
||||||
it.rating_key: it.id
|
it.rating_key: it.id
|
||||||
@@ -375,14 +444,27 @@ def reorder_playlist(
|
|||||||
select(PlexItem).where(PlexItem.rating_key.in_(order or [""]))
|
select(PlexItem).where(PlexItem.rating_key.in_(order or [""]))
|
||||||
).scalars()
|
).scalars()
|
||||||
}
|
}
|
||||||
for pos, rk in enumerate(order):
|
rows = {
|
||||||
iid = id_by_rk.get(rk)
|
r.item_id: r
|
||||||
if iid is not None:
|
for r in db.execute(
|
||||||
db.execute(
|
select(PlexPlaylistItem).where(PlexPlaylistItem.playlist_id == pid)
|
||||||
update(PlexPlaylistItem)
|
).scalars()
|
||||||
.where(PlexPlaylistItem.playlist_id == pid, PlexPlaylistItem.item_id == iid)
|
}
|
||||||
.values(position=pos)
|
pos = 0
|
||||||
)
|
seen: set[int] = set()
|
||||||
|
for rk in order:
|
||||||
|
row = rows.get(id_by_rk.get(rk, -1))
|
||||||
|
if row is not None and row.item_id not in seen:
|
||||||
|
row.position = pos
|
||||||
|
pos += 1
|
||||||
|
seen.add(row.item_id)
|
||||||
|
# Rows the payload omitted (added in another tab meanwhile) keep their relative order after
|
||||||
|
# the given ones, so no two rows share a position (the YouTube reorder does the same).
|
||||||
|
for row in sorted(
|
||||||
|
(r for iid, r in rows.items() if iid not in seen), key=lambda r: (r.position, r.id)
|
||||||
|
):
|
||||||
|
row.position = pos
|
||||||
|
pos += 1
|
||||||
pl.updated_at = datetime.now(timezone.utc)
|
pl.updated_at = datetime.now(timezone.utc)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"ok": True}
|
return {"ok": True}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel, field_validator
|
||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import func, select, update
|
||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
@@ -53,11 +53,30 @@ def list_views(
|
|||||||
return [_serialize(v) for v in rows]
|
return [_serialize(v) for v in rows]
|
||||||
|
|
||||||
|
|
||||||
|
# A feed view's tag pick is read by more than the feed: the autotag job keeps every tag a view names
|
||||||
|
# (sync/autotag.py). Held here to the shape the feed sends — a list of tag ids, at most this many — so
|
||||||
|
# a hand-made body (`"tags": 5`, or 60k ids) can't break that job for every user (E32 review).
|
||||||
|
_MAX_VIEW_TAGS = 500
|
||||||
|
|
||||||
|
|
||||||
|
def _check_filters(filters: dict | None) -> dict | None:
|
||||||
|
tags = (filters or {}).get("tags")
|
||||||
|
if tags is None:
|
||||||
|
return filters
|
||||||
|
if not isinstance(tags, list) or len(tags) > _MAX_VIEW_TAGS:
|
||||||
|
raise ValueError(f"filters.tags must be a list of at most {_MAX_VIEW_TAGS} tag ids")
|
||||||
|
if any(type(t) is not int for t in tags): # `type`, not isinstance: True is not a tag id
|
||||||
|
raise ValueError("filters.tags must hold tag ids")
|
||||||
|
return filters
|
||||||
|
|
||||||
|
|
||||||
class CreateViewIn(BaseModel):
|
class CreateViewIn(BaseModel):
|
||||||
name: str
|
name: str
|
||||||
filters: dict
|
filters: dict
|
||||||
source: str = "feed"
|
source: str = "feed"
|
||||||
|
|
||||||
|
_filters = field_validator("filters")(_check_filters)
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
@router.post("")
|
||||||
def create_view(
|
def create_view(
|
||||||
@@ -87,7 +106,7 @@ def create_view(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise refused(409, "view_name_taken", name=name)
|
raise refused(409, "view_name_taken", name=name) from None
|
||||||
return _serialize(view)
|
return _serialize(view)
|
||||||
|
|
||||||
|
|
||||||
@@ -98,6 +117,8 @@ class UpdateViewIn(BaseModel):
|
|||||||
filters: dict | None = None
|
filters: dict | None = None
|
||||||
is_default: bool | None = None
|
is_default: bool | None = None
|
||||||
|
|
||||||
|
_filters = field_validator("filters")(_check_filters)
|
||||||
|
|
||||||
|
|
||||||
@router.patch("/{view_id}")
|
@router.patch("/{view_id}")
|
||||||
def update_view(
|
def update_view(
|
||||||
@@ -135,7 +156,7 @@ def update_view(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise refused(409, "view_name_taken", name=view.name)
|
raise refused(409, "view_name_taken", name=view.name) from None
|
||||||
return _serialize(view)
|
return _serialize(view)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ def run_job_now(
|
|||||||
try:
|
try:
|
||||||
result = trigger_job(job_id, actor_id=user.id)
|
result = trigger_job(job_id, actor_id=user.id)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise not_found("job")
|
raise not_found("job") from None
|
||||||
if result == "already_running":
|
if result == "already_running":
|
||||||
raise refused(409, "job_running")
|
raise refused(409, "job_running")
|
||||||
return {"ok": True, "id": job_id, "started": True}
|
return {"ok": True, "id": job_id, "started": True}
|
||||||
|
|||||||
@@ -130,7 +130,7 @@ def _ingest_candidates(
|
|||||||
try:
|
try:
|
||||||
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
|
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
|
||||||
details = yt.get_channels(new_channel_ids)
|
details = yt.get_channels(new_channel_ids)
|
||||||
apply_channel_details(db, details)
|
apply_channel_details(db, details, new_channel_ids)
|
||||||
db.commit()
|
db.commit()
|
||||||
except YouTubeError:
|
except YouTubeError:
|
||||||
# The stubs remain and the details can be backfilled later — but until they are, those
|
# The stubs remain and the details can be backfilled later — but until they are, those
|
||||||
@@ -331,7 +331,7 @@ def search_youtube(
|
|||||||
# bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1).
|
# bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1).
|
||||||
try:
|
try:
|
||||||
collected.extend(_ingest_candidates(db, yt, user, candidates))
|
collected.extend(_ingest_candidates(db, yt, user, candidates))
|
||||||
except Exception as exc: # noqa: BLE001 — any ingest failure, not just YouTubeError
|
except Exception as exc: # any ingest failure, not just YouTubeError
|
||||||
db.rollback()
|
db.rollback()
|
||||||
# The term is NOT in this line. A search string is exactly what the privacy
|
# The term is NOT in this line. A search string is exactly what the privacy
|
||||||
# policy keeps out of the log unless the user turned Teljes/TRACE on for
|
# policy keeps out of the log unless the user turned Teljes/TRACE on for
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ def sync_subscriptions(user: User = Depends(require_human), db: Session = Depend
|
|||||||
# The onboarding wizard's first call: an unguarded failure here was a raw 500 and the
|
# The onboarding wizard's first call: an unguarded failure here was a raw 500 and the
|
||||||
# blocking "Server error" dialog, with nothing the user could act on (E9 F1).
|
# blocking "Server error" dialog, with nothing the user could act on (E9 F1).
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
result["quota_used_estimate"] = m.used
|
result["quota_used_estimate"] = m.used
|
||||||
result["quota_remaining_today"] = m.remaining
|
result["quota_remaining_today"] = m.remaining
|
||||||
return {"ok": True, **result}
|
return {"ok": True, **result}
|
||||||
@@ -71,7 +71,7 @@ def sync_status(user: User = Depends(current_user), db: Session = Depends(get_db
|
|||||||
def my_status(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
def my_status(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||||
"""Per-user sync progress: how far the channels *this* user subscribes to have synced."""
|
"""Per-user sync progress: how far the channels *this* user subscribes to have synced."""
|
||||||
sub_join = and_(Subscription.channel_id == Channel.id, Subscription.user_id == user.id)
|
sub_join = and_(Subscription.channel_id == Channel.id, Subscription.user_id == user.id)
|
||||||
total, details_synced, recent_synced, deep_done = db.execute(
|
total, details_synced, recent_synced, all_deep_done = db.execute(
|
||||||
select(
|
select(
|
||||||
func.count(Channel.id),
|
func.count(Channel.id),
|
||||||
func.count(Channel.details_synced_at),
|
func.count(Channel.details_synced_at),
|
||||||
@@ -82,7 +82,7 @@ def my_status(user: User = Depends(current_user), db: Session = Depends(get_db))
|
|||||||
.join(Subscription, sub_join)
|
.join(Subscription, sub_join)
|
||||||
).one()
|
).one()
|
||||||
total = total or 0
|
total = total or 0
|
||||||
deep_done = int(deep_done or 0)
|
all_deep_done = int(all_deep_done or 0)
|
||||||
|
|
||||||
# Per-user full-history (deep) backfill: channels this user has opted in, and an ETA
|
# Per-user full-history (deep) backfill: channels this user has opted in, and an ETA
|
||||||
# for the still-pending ones (quota-bound estimate).
|
# for the still-pending ones (quota-bound estimate).
|
||||||
@@ -97,6 +97,10 @@ def my_status(user: User = Depends(current_user), db: Session = Depends(get_db))
|
|||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
deep_eta = estimate_deep_backfill(db, deep_requested_channels)
|
deep_eta = estimate_deep_backfill(db, deep_requested_channels)
|
||||||
|
# "done / requested" is shown as one ratio, so both halves count the SAME channels: the ones this
|
||||||
|
# user asked full history for. Counting every subscribed channel whose back-catalog happens to be
|
||||||
|
# complete (another user asked, or it was short) gave "342/329".
|
||||||
|
deep_done = sum(1 for c in deep_requested_channels if c.backfill_done)
|
||||||
|
|
||||||
my_videos = db.scalar(
|
my_videos = db.scalar(
|
||||||
select(func.count(Video.id)).join(
|
select(func.count(Video.id)).join(
|
||||||
@@ -117,7 +121,7 @@ def my_status(user: User = Depends(current_user), db: Session = Depends(get_db))
|
|||||||
"channels_recent_synced": recent_synced,
|
"channels_recent_synced": recent_synced,
|
||||||
"channels_deep_done": deep_done,
|
"channels_deep_done": deep_done,
|
||||||
"channels_recent_pending": total - recent_synced,
|
"channels_recent_pending": total - recent_synced,
|
||||||
"channels_deep_pending": total - deep_done,
|
"channels_deep_pending": total - all_deep_done,
|
||||||
"channels_deep_requested": len(deep_requested_channels),
|
"channels_deep_requested": len(deep_requested_channels),
|
||||||
"deep_pending_count": deep_eta["channels_pending"],
|
"deep_pending_count": deep_eta["channels_pending"],
|
||||||
"deep_eta_seconds": deep_eta["eta_seconds"],
|
"deep_eta_seconds": deep_eta["eta_seconds"],
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ def create_tag(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise refused(409, "tag_name_taken", name=name)
|
raise refused(409, "tag_name_taken", name=name) from None
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"id": tag.id,
|
"id": tag.id,
|
||||||
@@ -113,7 +113,7 @@ def update_tag(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise refused(409, "tag_name_taken", name=tag.name)
|
raise refused(409, "tag_name_taken", name=tag.name) from None
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"id": tag.id,
|
"id": tag.id,
|
||||||
|
|||||||
+67
-11
@@ -18,6 +18,7 @@ from app.downloads.gc import run_download_gc
|
|||||||
from app.models import SchedulerSetting, User
|
from app.models import SchedulerSetting, User
|
||||||
from app.plex.sync import sync as run_plex_sync
|
from app.plex.sync import sync as run_plex_sync
|
||||||
from app.plex.tmdb import run_enrich as run_tmdb_enrich
|
from app.plex.tmdb import run_enrich as run_tmdb_enrich
|
||||||
|
from app.plex.tmdb import run_people_enrich
|
||||||
from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync
|
from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync
|
||||||
from app.notifications import create_notification
|
from app.notifications import create_notification
|
||||||
from app.state import is_sync_paused
|
from app.state import is_sync_paused
|
||||||
@@ -69,6 +70,7 @@ JOB_INTERVALS: dict[str, int] = {
|
|||||||
"plex_watch_sync": settings.plex_watch_sync_interval_min,
|
"plex_watch_sync": settings.plex_watch_sync_interval_min,
|
||||||
"plex_watch_reconcile": settings.plex_watch_reconcile_interval_min,
|
"plex_watch_reconcile": settings.plex_watch_reconcile_interval_min,
|
||||||
"tmdb_enrich": settings.tmdb_enrich_interval_min,
|
"tmdb_enrich": settings.tmdb_enrich_interval_min,
|
||||||
|
"people_enrich": settings.people_enrich_interval_min,
|
||||||
"audit_gc": settings.audit_gc_minutes,
|
"audit_gc": settings.audit_gc_minutes,
|
||||||
"log_gc": settings.log_gc_minutes,
|
"log_gc": settings.log_gc_minutes,
|
||||||
}
|
}
|
||||||
@@ -115,7 +117,9 @@ def _record(name: str, **fields) -> None:
|
|||||||
_activity.setdefault(name, {}).update(fields)
|
_activity.setdefault(name, {}).update(fields)
|
||||||
|
|
||||||
|
|
||||||
def _notify_done(db, actor_id: int, job_id: str, status: str, summary: str | None) -> None:
|
def _notify_done(
|
||||||
|
db, actor_id: int, job_id: str, status: str, summary: str | None, result: dict | None = None
|
||||||
|
) -> None:
|
||||||
"""Post a completion notice to the triggering admin's inbox (manual runs only). Best
|
"""Post a completion notice to the triggering admin's inbox (manual runs only). Best
|
||||||
effort — a notification failure must never mask the job's own outcome."""
|
effort — a notification failure must never mask the job's own outcome."""
|
||||||
try:
|
try:
|
||||||
@@ -125,7 +129,7 @@ def _notify_done(db, actor_id: int, job_id: str, status: str, summary: str | Non
|
|||||||
type="scheduler",
|
type="scheduler",
|
||||||
title=f"{job_id}: {status}", # English fallback; UI renders translated from data
|
title=f"{job_id}: {status}", # English fallback; UI renders translated from data
|
||||||
body=summary,
|
body=summary,
|
||||||
data={"job_id": job_id, "status": status, "summary": summary},
|
data={"job_id": job_id, "status": status, "summary": summary, "result": result},
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
@@ -140,7 +144,12 @@ def _run_changed(result) -> bool:
|
|||||||
if result is None:
|
if result is None:
|
||||||
return False
|
return False
|
||||||
if isinstance(result, dict):
|
if isinstance(result, dict):
|
||||||
return any(isinstance(v, (int, float)) and v for v in result.values())
|
# A phased job (backfill's recent/deep, people_enrich's bios/portraits) nests a result per
|
||||||
|
# phase: a count inside a phase is a change too.
|
||||||
|
return any(
|
||||||
|
_run_changed(v) if isinstance(v, dict) else isinstance(v, (int, float)) and bool(v)
|
||||||
|
for v in result.values()
|
||||||
|
)
|
||||||
if isinstance(result, (int, float)):
|
if isinstance(result, (int, float)):
|
||||||
return result != 0
|
return result != 0
|
||||||
return bool(result)
|
return bool(result)
|
||||||
@@ -152,6 +161,9 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
|||||||
they changed something, so routine no-op polls don't spam the trail. Best-effort."""
|
they changed something, so routine no-op polls don't spam the trail. Best-effort."""
|
||||||
if actor_id is None and status == "ok" and not _run_changed(result):
|
if actor_id is None and status == "ok" and not _run_changed(result):
|
||||||
return
|
return
|
||||||
|
# The result as data too (E30): the audit log names it in the reader's language, as the inbox
|
||||||
|
# does; `summary` stays the English line of the row's own summary column.
|
||||||
|
data = _result_data(result)
|
||||||
try:
|
try:
|
||||||
audit.record(
|
audit.record(
|
||||||
db,
|
db,
|
||||||
@@ -160,7 +172,7 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
|||||||
target_type="job",
|
target_type="job",
|
||||||
target_id=job_id,
|
target_id=job_id,
|
||||||
summary=f"{job_id}: {status}" + (f" — {summary}" if summary else ""),
|
summary=f"{job_id}: {status}" + (f" — {summary}" if summary else ""),
|
||||||
after={"status": status, "summary": summary},
|
after={"status": status, "summary": summary, **({"result": data} if data else {})},
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -192,28 +204,49 @@ def _job(name: str, fn) -> None:
|
|||||||
status="skipped",
|
status="skipped",
|
||||||
last_finished=_now_iso(),
|
last_finished=_now_iso(),
|
||||||
last_result="paused",
|
last_result="paused",
|
||||||
|
result=_PAUSED,
|
||||||
progress=None,
|
progress=None,
|
||||||
)
|
)
|
||||||
if actor_id is not None:
|
if actor_id is not None:
|
||||||
_notify_done(db, actor_id, name, "skipped", "sync paused")
|
_notify_done(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
||||||
_audit_run(db, actor_id, name, "skipped", "sync paused", None)
|
_audit_run(db, actor_id, name, "skipped", "sync paused", _PAUSED)
|
||||||
return
|
return
|
||||||
with progress.bind(sink):
|
with progress.bind(sink):
|
||||||
result = fn(db)
|
result = fn(db)
|
||||||
summary = _summarize(result)
|
summary = _summarize(result)
|
||||||
|
data = _result_data(result)
|
||||||
log.info("job %s -> %s", name, result)
|
log.info("job %s -> %s", name, result)
|
||||||
_record(name, status="ok", last_finished=_now_iso(), last_result=summary, progress=None)
|
_record(
|
||||||
|
name,
|
||||||
|
status="ok",
|
||||||
|
last_finished=_now_iso(),
|
||||||
|
last_result=summary,
|
||||||
|
result=data,
|
||||||
|
progress=None,
|
||||||
|
)
|
||||||
if actor_id is not None:
|
if actor_id is not None:
|
||||||
_notify_done(db, actor_id, name, "ok", summary)
|
_notify_done(db, actor_id, name, "ok", summary, data)
|
||||||
_audit_run(db, actor_id, name, "ok", summary, result)
|
_audit_run(db, actor_id, name, "ok", summary, result)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
|
log.exception("job %s failed", name, extra=logs.event("scheduler.job.failed", job=name))
|
||||||
err = str(exc) or exc.__class__.__name__
|
err = str(exc) or exc.__class__.__name__
|
||||||
_record(name, status="error", last_finished=_now_iso(), last_error=err, progress=None)
|
# A failure the job named (progress.JobFailed) rides along as data, as a result does, so the
|
||||||
|
# client says it in the reader's language; `err` stays the English line.
|
||||||
|
failure = (
|
||||||
|
{"failure": exc.reason, **exc.params} if isinstance(exc, progress.JobFailed) else None
|
||||||
|
)
|
||||||
|
_record(
|
||||||
|
name,
|
||||||
|
status="error",
|
||||||
|
last_finished=_now_iso(),
|
||||||
|
last_error=err,
|
||||||
|
result=failure,
|
||||||
|
progress=None,
|
||||||
|
)
|
||||||
if actor_id is not None:
|
if actor_id is not None:
|
||||||
_notify_done(db, actor_id, name, "error", err)
|
_notify_done(db, actor_id, name, "error", err, failure)
|
||||||
_audit_run(db, actor_id, name, "error", err, None)
|
_audit_run(db, actor_id, name, "error", err, failure)
|
||||||
finally:
|
finally:
|
||||||
# ALWAYS release the claim, even if the body raised before its own status update — so a
|
# ALWAYS release the claim, even if the body raised before its own status update — so a
|
||||||
# failure can never wedge the job as permanently "running".
|
# failure can never wedge the job as permanently "running".
|
||||||
@@ -221,6 +254,21 @@ def _job(name: str, fn) -> None:
|
|||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
|
||||||
|
# A skipped run's result, in the same shape as a job's own `{"skipped": …}`.
|
||||||
|
_PAUSED = {"skipped": "sync paused"}
|
||||||
|
|
||||||
|
|
||||||
|
def _result_data(result) -> dict | None:
|
||||||
|
"""A job's result as the client names it in the reader's language (E30): the dict itself (counts,
|
||||||
|
a `skipped` / `reason` word, a nested dict per phase), `enrich`'s bare count as one. The one-line
|
||||||
|
`_summarize` is the English fallback."""
|
||||||
|
if isinstance(result, dict):
|
||||||
|
return result or None
|
||||||
|
if isinstance(result, int) and not isinstance(result, bool):
|
||||||
|
return {"enriched": result}
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _summarize(result) -> str | None:
|
def _summarize(result) -> str | None:
|
||||||
"""Compact one-line rendering of a job's return value for the dashboard."""
|
"""Compact one-line rendering of a job's return value for the dashboard."""
|
||||||
if result is None:
|
if result is None:
|
||||||
@@ -259,6 +307,7 @@ def scheduler_snapshot() -> dict:
|
|||||||
"last_started": a.get("last_started"),
|
"last_started": a.get("last_started"),
|
||||||
"last_finished": a.get("last_finished"),
|
"last_finished": a.get("last_finished"),
|
||||||
"last_result": a.get("last_result"),
|
"last_result": a.get("last_result"),
|
||||||
|
"result": a.get("result"),
|
||||||
"last_error": a.get("last_error"),
|
"last_error": a.get("last_error"),
|
||||||
"progress": a.get("progress"),
|
"progress": a.get("progress"),
|
||||||
}
|
}
|
||||||
@@ -369,6 +418,12 @@ def _tmdb_enrich_job() -> None:
|
|||||||
_job("tmdb_enrich", run_tmdb_enrich)
|
_job("tmdb_enrich", run_tmdb_enrich)
|
||||||
|
|
||||||
|
|
||||||
|
def _people_enrich_job() -> None:
|
||||||
|
# Make the people local (E30 S6): a batch of person bios from TMDB and a batch of portraits resized
|
||||||
|
# through Plex, so a person page and a cast strip never wait on an outside service.
|
||||||
|
_job("people_enrich", run_people_enrich)
|
||||||
|
|
||||||
|
|
||||||
def _audit_gc_job() -> None:
|
def _audit_gc_job() -> None:
|
||||||
# Prune audit-log rows older than the admin-set retention (audit_retention_days; 0 = keep
|
# Prune audit-log rows older than the admin-set retention (audit_retention_days; 0 = keep
|
||||||
# forever). Pure DB, no quota.
|
# forever). Pure DB, no quota.
|
||||||
@@ -415,6 +470,7 @@ JOB_FUNCS: dict[str, Callable[[], None]] = {
|
|||||||
"plex_watch_sync": _plex_watch_sync_job,
|
"plex_watch_sync": _plex_watch_sync_job,
|
||||||
"plex_watch_reconcile": _plex_watch_reconcile_job,
|
"plex_watch_reconcile": _plex_watch_reconcile_job,
|
||||||
"tmdb_enrich": _tmdb_enrich_job,
|
"tmdb_enrich": _tmdb_enrich_job,
|
||||||
|
"people_enrich": _people_enrich_job,
|
||||||
"audit_gc": _audit_gc_job,
|
"audit_gc": _audit_gc_job,
|
||||||
"log_gc": _log_gc_job,
|
"log_gc": _log_gc_job,
|
||||||
}
|
}
|
||||||
|
|||||||
+32
-13
@@ -12,7 +12,7 @@ from sqlalchemy import and_, exists, func, select
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs, progress, sysconfig
|
from app import logs, progress, sysconfig
|
||||||
from app.models import Channel, ChannelTag, Tag, Video
|
from app.models import Channel, ChannelTag, SavedView, Tag, Video
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
@@ -115,7 +115,9 @@ _TOPIC_SLUGS = {
|
|||||||
"Tourism": "Travel",
|
"Tourism": "Travel",
|
||||||
"Fashion": "Fashion & Beauty",
|
"Fashion": "Fashion & Beauty",
|
||||||
"Physical_attractiveness": "Fashion & Beauty",
|
"Physical_attractiveness": "Fashion & Beauty",
|
||||||
"Humor": "Comedy",
|
# YouTube links the British page (…/wiki/Humour); the key was "Humor", so no channel ever got Comedy
|
||||||
|
# from its topic (E30).
|
||||||
|
"Humour": "Comedy",
|
||||||
"Entertainment": "Entertainment",
|
"Entertainment": "Entertainment",
|
||||||
"Film": "Film",
|
"Film": "Film",
|
||||||
"Television_program": "TV",
|
"Television_program": "TV",
|
||||||
@@ -286,18 +288,35 @@ def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
|
|||||||
return {"channels_tagged": tagged, "orphan_tags_removed": removed}
|
return {"channels_tagged": tagged, "orphan_tags_removed": removed}
|
||||||
|
|
||||||
|
|
||||||
|
def _view_tag_ids(filters: object) -> list[int]:
|
||||||
|
"""The tag ids a stored view's filters name; anything else in that slot names none."""
|
||||||
|
tags = filters.get("tags") if isinstance(filters, dict) else None
|
||||||
|
if not isinstance(tags, list):
|
||||||
|
return []
|
||||||
|
return [t for t in tags if type(t) is int] # `type`, not isinstance: True is not a tag id
|
||||||
|
|
||||||
|
|
||||||
def _cleanup_orphan_system_tags(db: Session) -> int:
|
def _cleanup_orphan_system_tags(db: Session) -> int:
|
||||||
"""Delete system tags that no longer link to any channel (e.g. after a mapping change)."""
|
"""Delete system tags that no longer link to any channel (e.g. after a mapping change) — but not
|
||||||
orphans = (
|
one a saved feed view filters on. Views hold tag IDS, and a deleted tag comes back under a new id
|
||||||
db.execute(
|
when a channel earns it again, so the view's filter would point at nothing; the feed drops an
|
||||||
select(Tag).where(
|
unknown id, and the view showed the whole feed under its old name (E32 S2).
|
||||||
Tag.user_id.is_(None),
|
|
||||||
~exists().where(ChannelTag.tag_id == Tag.id),
|
The views are read defensively, and the kept ids are filtered here rather than in SQL: a view
|
||||||
)
|
stored before its route checked the shape (or written around it) must not raise for everyone's
|
||||||
)
|
job, nor push a `NOT IN` past the driver's bind-parameter limit (E32 review)."""
|
||||||
.scalars()
|
in_views = {
|
||||||
.all()
|
t
|
||||||
)
|
for (filters,) in db.execute(select(SavedView.filters).where(SavedView.source == "feed"))
|
||||||
|
for t in _view_tag_ids(filters)
|
||||||
|
}
|
||||||
|
orphans = [
|
||||||
|
tag
|
||||||
|
for tag in db.execute(
|
||||||
|
select(Tag).where(Tag.user_id.is_(None), ~exists().where(ChannelTag.tag_id == Tag.id))
|
||||||
|
).scalars()
|
||||||
|
if tag.id not in in_views
|
||||||
|
]
|
||||||
for tag in orphans:
|
for tag in orphans:
|
||||||
db.delete(tag)
|
db.delete(tag)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ from app.models import (
|
|||||||
)
|
)
|
||||||
from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
|
from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
|
||||||
from app.utils import now_utc as _now
|
from app.utils import now_utc as _now
|
||||||
from app.youtube.client import YouTubeClient
|
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
@@ -81,7 +81,15 @@ def explore_ingest_page(
|
|||||||
the organic deep backfill) is never touched."""
|
the organic deep backfill) is never touched."""
|
||||||
if not channel.uploads_playlist_id:
|
if not channel.uploads_playlist_id:
|
||||||
return {"ingested": 0, "next_page_token": None}
|
return {"ingested": 0, "next_page_token": None}
|
||||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
try:
|
||||||
|
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||||
|
except YouTubeError as exc:
|
||||||
|
# A channel with no uploads has an uploads id that YouTube answers as not found: an empty
|
||||||
|
# back-catalogue, not a failure (and not a "playlist gone" — the user picked a channel).
|
||||||
|
if not is_playlist_gone(exc):
|
||||||
|
raise
|
||||||
|
log.info("explore ingest channel=%s has no uploads playlist", channel.id)
|
||||||
|
return {"ingested": 0, "next_page_token": None}
|
||||||
stubs: list[dict] = []
|
stubs: list[dict] = []
|
||||||
for item in data.get("items", []):
|
for item in data.get("items", []):
|
||||||
stub = stub_from_playlist_item(item, channel.id)
|
stub = stub_from_playlist_item(item, channel.id)
|
||||||
|
|||||||
@@ -20,9 +20,9 @@ Quota: phase 1 costs ~(flagged / 50) units, phase 2 ~(batch / 50). Both respect
|
|||||||
backfill reserve as the other jobs so interactive syncs never starve.
|
backfill reserve as the other jobs so interactive syncs never starve.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import timedelta
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs, progress, quota, sysconfig
|
from app import logs, progress, quota, sysconfig
|
||||||
@@ -129,14 +129,14 @@ def _notify_removed(db: Session, impact: dict[int, list[dict]]) -> int:
|
|||||||
return len(impact)
|
return len(impact)
|
||||||
|
|
||||||
|
|
||||||
def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
|
def _recheck_flagged(db: Session, yt: YouTubeClient, now: datetime) -> dict:
|
||||||
"""Phase 1: recover or hard-delete videos already flagged unavailable."""
|
"""Phase 1: recover or hard-delete videos already flagged unavailable. `now` is the run's:
|
||||||
|
phase 2 skips a video stamped with it, so one recovered here is not fetched again."""
|
||||||
flagged = db.execute(select(Video).where(Video.unavailable_since.is_not(None))).scalars().all()
|
flagged = db.execute(select(Video).where(Video.unavailable_since.is_not(None))).scalars().all()
|
||||||
if not flagged:
|
if not flagged:
|
||||||
return {"checked": 0, "recovered": 0, "deleted": 0, "notified": 0}
|
return {"checked": 0, "recovered": 0, "deleted": 0, "notified": 0}
|
||||||
progress.report(0, len(flagged), "recheck")
|
progress.report(0, len(flagged), "recheck")
|
||||||
items = {it["id"]: it for it in yt.get_videos([v.id for v in flagged])}
|
items = {it["id"]: it for it in yt.get_videos([v.id for v in flagged])}
|
||||||
now = _now()
|
|
||||||
recovered = 0
|
recovered = 0
|
||||||
to_delete: list[Video] = []
|
to_delete: list[Video] = []
|
||||||
for video in flagged:
|
for video in flagged:
|
||||||
@@ -173,7 +173,7 @@ def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
def _revalidate_rolling(db: Session, yt: YouTubeClient, now: datetime) -> dict:
|
||||||
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
|
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
|
||||||
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
|
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
|
||||||
batch = sysconfig.get_int(
|
batch = sysconfig.get_int(
|
||||||
@@ -181,7 +181,6 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
|||||||
) # admin override, else config default
|
) # admin override, else config default
|
||||||
flagged_new = 0
|
flagged_new = 0
|
||||||
checked = 0
|
checked = 0
|
||||||
now = _now()
|
|
||||||
# Process in videos.list-sized pages so we commit incrementally and stop on low quota.
|
# Process in videos.list-sized pages so we commit incrementally and stop on low quota.
|
||||||
while checked < batch:
|
while checked < batch:
|
||||||
if quota.backfill_reserve_reached():
|
if quota.backfill_reserve_reached():
|
||||||
@@ -190,6 +189,10 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
|||||||
db.execute(
|
db.execute(
|
||||||
select(Video)
|
select(Video)
|
||||||
.where(Video.unavailable_since.is_(None))
|
.where(Video.unavailable_since.is_(None))
|
||||||
|
# Not one this run already checked: with fewer available videos than the batch, the
|
||||||
|
# loop used to wrap around and re-fetch the same ones until `checked` reached the
|
||||||
|
# batch, spending quota on answers it already had (E32 S2).
|
||||||
|
.where(or_(Video.last_checked_at.is_(None), Video.last_checked_at < now))
|
||||||
# NULLS FIRST = never-checked first, then least-recently-checked; equivalent to the old
|
# NULLS FIRST = never-checked first, then least-recently-checked; equivalent to the old
|
||||||
# `(last_checked_at IS NULL) DESC, last_checked_at ASC` but in the exact shape of the
|
# `(last_checked_at IS NULL) DESC, last_checked_at ASC` but in the exact shape of the
|
||||||
# partial ix_videos_recheck_order (migration 0069, PERF-18) so it rides that index.
|
# partial ix_videos_recheck_order (migration 0069, PERF-18) so it rides that index.
|
||||||
@@ -220,10 +223,13 @@ def run_maintenance(db: Session) -> dict:
|
|||||||
user = get_service_user(db)
|
user = get_service_user(db)
|
||||||
if user is None:
|
if user is None:
|
||||||
return {"reason": "no service user"}
|
return {"reason": "no service user"}
|
||||||
|
# One `now` for the whole run: each phase taking its own let a video phase 1 had just recovered
|
||||||
|
# (stamped with phase 1's) pass phase 2's "not checked this run" test and be fetched twice.
|
||||||
|
now = _now()
|
||||||
with YouTubeClient(db, user) as yt:
|
with YouTubeClient(db, user) as yt:
|
||||||
phase1 = _recheck_flagged(db, yt)
|
phase1 = _recheck_flagged(db, yt, now)
|
||||||
phase2 = (
|
phase2 = (
|
||||||
_revalidate_rolling(db, yt)
|
_revalidate_rolling(db, yt, now)
|
||||||
if not quota.backfill_reserve_reached()
|
if not quota.backfill_reserve_reached()
|
||||||
else {"checked": 0, "flagged": 0, "skipped": "low quota"}
|
else {"checked": 0, "flagged": 0, "skipped": "low quota"}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
"""The 3-way merge of a YouTube-linked playlist, as a pure function: what each side gains and loses
|
||||||
|
so neither loses an item, the merged order, and the reorder moves YouTube needs to reach it. No I/O —
|
||||||
|
`sync/playlists.py` reads both sides, calls `merge_plan`, and writes the outcome."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
def _reorder_moves(current: list[str], desired: list[str]) -> int:
|
||||||
|
"""Count the playlistItems.update moves an insertion-sort would need to turn
|
||||||
|
`current` order into `desired` order (both already the same set of video ids).
|
||||||
|
Items already in place are skipped; this matches what apply_push actually executes."""
|
||||||
|
model = list(current)
|
||||||
|
moves = 0
|
||||||
|
for i, want in enumerate(desired):
|
||||||
|
if i < len(model) and model[i] == want:
|
||||||
|
continue
|
||||||
|
j = model.index(want, i)
|
||||||
|
model.pop(j)
|
||||||
|
model.insert(i, want)
|
||||||
|
moves += 1
|
||||||
|
return moves
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MergePlan:
|
||||||
|
"""The 3-way merge of a linked playlist: what each side gains and loses so neither side
|
||||||
|
loses an item, plus the merged order. `final` is the id set both sides end up with."""
|
||||||
|
|
||||||
|
final: list[str] # merged order (local order, YouTube-only adds appended)
|
||||||
|
add_to_yt: list[str] # local-adds to push out to YouTube
|
||||||
|
add_to_local: list[str] # YouTube-adds to pull into Siftlode
|
||||||
|
remove_from_yt: list[str] # items deleted locally that are still on YouTube
|
||||||
|
remove_from_local: list[str] # items deleted on YouTube that are still local
|
||||||
|
reorders: int # estimated YouTube playlistItems.update moves to reach `final`
|
||||||
|
baseline_known: bool
|
||||||
|
|
||||||
|
|
||||||
|
def merge_plan(
|
||||||
|
baseline: list[str] | None,
|
||||||
|
local: list[str],
|
||||||
|
youtube: list[str],
|
||||||
|
unavailable: set[str] | None = None,
|
||||||
|
) -> MergePlan:
|
||||||
|
"""Pure 3-way merge. Classify each id against the last-synced `baseline`:
|
||||||
|
- in local but not youtube, not deleted → add to YouTube (a local-add)
|
||||||
|
- in youtube but not local, not deleted → add to Siftlode (a YouTube-add)
|
||||||
|
- in baseline but gone from local → a genuine local delete → propagate to YouTube
|
||||||
|
- in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode
|
||||||
|
With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both
|
||||||
|
sides (never delete) — the lossless default. Merged order: local order first, YouTube-only
|
||||||
|
additions appended in their YouTube order.
|
||||||
|
`unavailable` = known un-ingestable-but-live YouTube ids: dropped from the YouTube side up
|
||||||
|
front so they're never re-classified as a YouTube-add (they can't ingest) nor as a local
|
||||||
|
delete (they were never local) — otherwise such a video would make the playlist never
|
||||||
|
converge. They stay in place on YouTube, untouched by this merge."""
|
||||||
|
un = set(unavailable or ())
|
||||||
|
L = list(dict.fromkeys(local))
|
||||||
|
Y = [v for v in dict.fromkeys(youtube) if v not in un]
|
||||||
|
Lset, Yset = set(L), set(Y)
|
||||||
|
if baseline is None:
|
||||||
|
add_to_yt = [v for v in L if v not in Yset]
|
||||||
|
add_to_local = [v for v in Y if v not in Lset]
|
||||||
|
remove_from_yt: list[str] = []
|
||||||
|
remove_from_local: list[str] = []
|
||||||
|
final = L + [v for v in Y if v not in Lset]
|
||||||
|
else:
|
||||||
|
B = set(baseline)
|
||||||
|
deleted_local = B - Lset # was synced, removed on our side
|
||||||
|
deleted_yt = B - Yset # was synced, removed on YouTube's side
|
||||||
|
gone = deleted_local | deleted_yt
|
||||||
|
remove_from_yt = [v for v in Y if v in deleted_local]
|
||||||
|
remove_from_local = [v for v in L if v in deleted_yt]
|
||||||
|
add_to_yt = [v for v in L if v not in Yset and v not in gone]
|
||||||
|
add_to_local = [v for v in Y if v not in Lset and v not in gone]
|
||||||
|
final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone]
|
||||||
|
final_set = set(final)
|
||||||
|
# On YouTube, after membership reconciliation the order is current-kept + inserts appended;
|
||||||
|
# the reorder pass then moves items into `final`.
|
||||||
|
after_membership = [v for v in Y if v in final_set] + add_to_yt
|
||||||
|
reorders = _reorder_moves(after_membership, final)
|
||||||
|
return MergePlan(
|
||||||
|
final=final,
|
||||||
|
add_to_yt=add_to_yt,
|
||||||
|
add_to_local=add_to_local,
|
||||||
|
remove_from_yt=remove_from_yt,
|
||||||
|
remove_from_local=remove_from_local,
|
||||||
|
reorders=reorders,
|
||||||
|
baseline_known=baseline is not None,
|
||||||
|
)
|
||||||
+220
-122
@@ -1,6 +1,7 @@
|
|||||||
"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists
|
"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists
|
||||||
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync
|
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync
|
||||||
local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope.
|
local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope; the
|
||||||
|
pure 3-way merge they run is `sync/playlist_merge.py`.
|
||||||
|
|
||||||
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
|
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
|
||||||
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
|
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
|
||||||
@@ -8,25 +9,110 @@ ingested (with a stub channel) so the mirror is faithful."""
|
|||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable, Iterator
|
||||||
from dataclasses import dataclass
|
from contextlib import AbstractContextManager, contextmanager
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import TypeVar
|
from typing import TypeVar
|
||||||
|
|
||||||
from sqlalchemy import delete, select
|
from sqlalchemy import Engine, create_engine, delete, func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
from sqlalchemy.pool import NullPool
|
||||||
|
|
||||||
from app import logs, progress, quota
|
from app import logs, progress, quota
|
||||||
from app.auth import has_read_scope
|
from app.auth import has_read_scope
|
||||||
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
||||||
|
from app.sync.playlist_merge import merge_plan
|
||||||
from app.sync.runner import token_users
|
from app.sync.runner import token_users
|
||||||
|
from app.sync.subscriptions import blank_listing, refuse_blank_listing, vanished
|
||||||
from app.sync.videos import apply_video_details, parse_dt
|
from app.sync.videos import apply_video_details, parse_dt
|
||||||
from app.youtube.client import YouTubeClient, YouTubeError
|
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
||||||
WRITE_UNIT_COST = 50
|
WRITE_UNIT_COST = 50
|
||||||
|
|
||||||
|
# The advisory-lock namespaces ("PL" per playlist, "PU" per user), so a key cannot meet another
|
||||||
|
# feature's lock on the same number (Postgres keeps the two-int keys apart from the one-bigint ones).
|
||||||
|
_SYNC_LOCK_NS = 0x504C
|
||||||
|
_USER_SYNC_LOCK_NS = 0x5055
|
||||||
|
|
||||||
|
# The claims' own connections, outside the app pool: a claim is held for a whole push (seconds of
|
||||||
|
# YouTube calls) beside the request's pooled session and quota's, so a burst of pushes drawing a
|
||||||
|
# third pooled connection each ran the pool dry and stalled every other request for pool_timeout.
|
||||||
|
# NullPool opens one per claim and closes it after, so a lock can never ride back into a pool either.
|
||||||
|
_claim_engines: dict[str, Engine] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def _claim_engine(db: Session) -> Engine:
|
||||||
|
url = db.get_bind().engine.url
|
||||||
|
key = url.render_as_string(hide_password=False)
|
||||||
|
eng = _claim_engines.get(key)
|
||||||
|
if eng is None:
|
||||||
|
eng = _claim_engines[key] = create_engine(url, poolclass=NullPool, hide_parameters=True)
|
||||||
|
return eng
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _advisory_claim(db: Session, ns: int, key: int) -> Iterator[bool]:
|
||||||
|
"""Try a session-level advisory lock for the block on a connection of its own; yields whether it
|
||||||
|
was free. It survives the block's commits, and a crashed worker's lock dies with its connection
|
||||||
|
— no claim column that a crash leaves stuck."""
|
||||||
|
conn = _claim_engine(db).connect()
|
||||||
|
held = False
|
||||||
|
try:
|
||||||
|
held = bool(conn.scalar(select(func.pg_try_advisory_lock(ns, key))))
|
||||||
|
conn.commit()
|
||||||
|
yield held
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
if held:
|
||||||
|
conn.scalar(select(func.pg_advisory_unlock(ns, key)))
|
||||||
|
conn.commit()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def sync_claim(db: Session, playlist_id: int) -> AbstractContextManager[bool]:
|
||||||
|
"""Claim one playlist's YouTube sync (push, re-pull, the scheduled mirror refresh) for the block;
|
||||||
|
yields whether it was free. A push reads YouTube, then writes both sides for seconds and commits
|
||||||
|
step by step, so a row lock ends at its first commit and two tabs (or a push and the scheduler)
|
||||||
|
interleave into a doubled YouTube item or a baseline recorded over the other's writes."""
|
||||||
|
return _advisory_claim(db, _SYNC_LOCK_NS, playlist_id)
|
||||||
|
|
||||||
|
|
||||||
|
class PlaylistGone(Exception):
|
||||||
|
"""The list a sync was working on was deleted meanwhile (another tab, the mirror drop). Its
|
||||||
|
items went with it, so a plan made now reads "every item removed here" — and would delete them
|
||||||
|
all from the YouTube playlist the reader chose to keep. The route answers 404."""
|
||||||
|
|
||||||
|
|
||||||
|
def lock_playlists(db: Session, *playlist_ids: int) -> set[int]:
|
||||||
|
"""Row-lock playlists before touching their items, several in id order. Every item write takes
|
||||||
|
the PLAYLIST first — the route edits, a push, a re-pull, the scheduled mirror refresh and a
|
||||||
|
delete alike: a writer that locked an item row first and the playlist second (a push's item
|
||||||
|
replace beside a remove) deadlocks against one that went the other way, and the loser is a
|
||||||
|
500. The rows are re-read, so `dirty`, the name and the sync baseline are the ones a finished
|
||||||
|
writer committed. Held to the caller's next commit — so no commit may come between this and
|
||||||
|
the write it guards. Returns the ids it found: a list deleted meanwhile is not among them."""
|
||||||
|
rows = db.scalars(
|
||||||
|
select(Playlist)
|
||||||
|
.where(Playlist.id.in_(sorted(set(playlist_ids))))
|
||||||
|
.order_by(Playlist.id)
|
||||||
|
.with_for_update()
|
||||||
|
.execution_options(populate_existing=True)
|
||||||
|
).all() # consumed: rows fetched are what refreshes the loaded objects
|
||||||
|
return {pl.id for pl in rows}
|
||||||
|
|
||||||
|
|
||||||
|
def _lock_or_gone(db: Session, pl: Playlist) -> None:
|
||||||
|
"""`lock_playlists` for a sync that goes on to write YouTube or the items: a list deleted since
|
||||||
|
it was read raises `PlaylistGone` (rolled back) instead of being planned as an empty one."""
|
||||||
|
pl_id = pl.id # read first: the rollback expires `pl`, and its row is gone
|
||||||
|
if pl_id not in lock_playlists(db, pl_id):
|
||||||
|
db.rollback()
|
||||||
|
raise PlaylistGone(pl_id)
|
||||||
|
|
||||||
|
|
||||||
def fingerprint(name: str, ids: list[str]) -> str:
|
def fingerprint(name: str, ids: list[str]) -> str:
|
||||||
"""A stable checksum of a playlist's name + ordered video ids. Stored as the synced
|
"""A stable checksum of a playlist's name + ordered video ids. Stored as the synced
|
||||||
@@ -91,11 +177,18 @@ def parse_unavailable(pl: Playlist) -> set[str]:
|
|||||||
return _load_baseline(pl)[1]
|
return _load_baseline(pl)[1]
|
||||||
|
|
||||||
|
|
||||||
def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None = None) -> None:
|
def _set_baseline(
|
||||||
|
pl: Playlist,
|
||||||
|
ids: list[str],
|
||||||
|
unavailable: list[str] | None = None,
|
||||||
|
*,
|
||||||
|
name: str | None = None,
|
||||||
|
) -> None:
|
||||||
"""Record `ids` as both the hash (drives `dirty`, over the local ids only) and the raw ordered
|
"""Record `ids` as both the hash (drives `dirty`, over the local ids only) and the raw ordered
|
||||||
set (drives merge). `unavailable` = un-ingestable-but-live YouTube ids to carry forward so the
|
set (drives merge). `unavailable` = un-ingestable-but-live YouTube ids to carry forward so the
|
||||||
next merge skips them; stored only when non-empty (keeps the common case a plain list)."""
|
next merge skips them; stored only when non-empty (keeps the common case a plain list). `name`:
|
||||||
pl.synced_fingerprint = fingerprint(pl.name, ids)
|
the title YouTube now has, when that may differ from the list's own (see the push)."""
|
||||||
|
pl.synced_fingerprint = fingerprint(pl.name if name is None else name, ids)
|
||||||
if unavailable:
|
if unavailable:
|
||||||
pl.synced_video_ids = json.dumps({"ids": ids, "unavailable": list(unavailable)})
|
pl.synced_video_ids = json.dumps({"ids": ids, "unavailable": list(unavailable)})
|
||||||
else:
|
else:
|
||||||
@@ -103,6 +196,25 @@ def _set_baseline(pl: Playlist, ids: list[str], unavailable: list[str] | None =
|
|||||||
pl.dirty = False
|
pl.dirty = False
|
||||||
|
|
||||||
|
|
||||||
|
def _set_baseline_under_lock(
|
||||||
|
db: Session,
|
||||||
|
pl: Playlist,
|
||||||
|
ids: list[str],
|
||||||
|
unavailable: list[str] | None = None,
|
||||||
|
*,
|
||||||
|
name: str,
|
||||||
|
) -> None:
|
||||||
|
"""A push's closing baseline: `ids` and `name` are what YouTube now holds. The push spent
|
||||||
|
seconds on YouTube writes since it replaced the local items, so an edit may have landed
|
||||||
|
meanwhile — lock the list and derive `dirty` from what is there, or the edit would read as
|
||||||
|
synced and never reach YouTube. The name too: the lock re-reads a rename made meanwhile, and a
|
||||||
|
baseline taken over THAT name matched it, so the list read clean with YouTube's title still the
|
||||||
|
old one (E28 review round 4)."""
|
||||||
|
_lock_or_gone(db, pl)
|
||||||
|
_set_baseline(pl, ids, unavailable, name=name)
|
||||||
|
recompute_dirty(db, pl)
|
||||||
|
|
||||||
|
|
||||||
def recompute_dirty(db: Session, pl: Playlist) -> None:
|
def recompute_dirty(db: Session, pl: Playlist) -> None:
|
||||||
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
|
"""Set `dirty` by comparing the current state to the synced baseline. Unlinked lists
|
||||||
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
|
(and Watch later) are never dirty; a missing baseline counts as dirty (unknown YT state)."""
|
||||||
@@ -159,15 +271,13 @@ def _ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None
|
|||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
def _replace_items_from_youtube(
|
def _replace_items_from_youtube(db: Session, pl: Playlist, ids: list[str]) -> list[str]:
|
||||||
db: Session, yt: YouTubeClient, pl: Playlist, ids: list[str]
|
|
||||||
) -> list[str]:
|
|
||||||
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
|
"""Mirror a YouTube playlist's ordered video ids into `pl`'s local items (authoritative
|
||||||
replace): ensure the videos exist in the catalog, keep only those present in their YT
|
replace): keep only those present in the catalog in their YT order (de-duplicated), swap the
|
||||||
order (de-duplicated), swap the items out, and reset the synced fingerprint + dirty flag.
|
items out, and reset the synced fingerprint + dirty flag. The caller has ingested `ids`
|
||||||
`pl.name` must already be set to the desired name. The caller commits. Returns the final
|
(`_ensure_videos`, which commits) and THEN taken `lock_playlists`, set `pl.name`, and commits
|
||||||
ordered ids."""
|
after: an ingest in here re-fetched every id YouTube will not return (a private or deleted
|
||||||
_ensure_videos(db, yt, ids)
|
video) and its commit let go of the lock before the replace. Returns the final ordered ids."""
|
||||||
present = (
|
present = (
|
||||||
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
|
set(db.execute(select(Video.id).where(Video.id.in_(ids))).scalars().all()) if ids else set()
|
||||||
)
|
)
|
||||||
@@ -196,6 +306,39 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
|||||||
the user's local playlists and the built-in Watch later untouched."""
|
the user's local playlists and the built-in Watch later untouched."""
|
||||||
if not has_read_scope(user):
|
if not has_read_scope(user):
|
||||||
return {"synced": 0, "reason": "no read scope"}
|
return {"synced": 0, "reason": "no read scope"}
|
||||||
|
with user_sync_claim(db, user.id) as free:
|
||||||
|
if not free:
|
||||||
|
return {"synced": 0, "reason": "busy"}
|
||||||
|
return pull_user_playlists(db, user)
|
||||||
|
|
||||||
|
|
||||||
|
def user_sync_claim(db: Session, user_id: int) -> AbstractContextManager[bool]:
|
||||||
|
"""Claim a user's bulk mirror pull for the block; yields whether it was free. Two runs at once
|
||||||
|
(the scheduler beside a manual "sync now", two tabs) both saw a YouTube list unmirrored and
|
||||||
|
each created its mirror — a duplicate row."""
|
||||||
|
return _advisory_claim(db, _USER_SYNC_LOCK_NS, user_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _mirror_listing(yt: YouTubeClient, ytid: str, count: object, pl_id: int) -> list[str] | None:
|
||||||
|
"""A mirror's video ids from YouTube, or None when there is nothing to believe: the list was
|
||||||
|
deleted between the pull's listing and this read (the next pull drops the mirror as vanished —
|
||||||
|
not a reason to fail every other list, E32 review round 2), or YouTube listed no items for a
|
||||||
|
list it counts as non-empty (`blank_listing`)."""
|
||||||
|
try:
|
||||||
|
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
||||||
|
except YouTubeError as exc:
|
||||||
|
if not is_playlist_gone(exc):
|
||||||
|
raise
|
||||||
|
log.info("YouTube playlist of %s vanished mid-pull; skipped", pl_id)
|
||||||
|
return None
|
||||||
|
if blank_listing(ids, count):
|
||||||
|
log.warning("YouTube listed no items for non-empty playlist %s; kept", pl_id)
|
||||||
|
return None
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
|
def pull_user_playlists(db: Session, user: User) -> dict:
|
||||||
|
"""The mirror pull itself; the caller holds `user_sync_claim`."""
|
||||||
synced = 0
|
synced = 0
|
||||||
with YouTubeClient(db, user) as yt:
|
with YouTubeClient(db, user) as yt:
|
||||||
yt_playlists = list(yt.iter_my_playlists())
|
yt_playlists = list(yt.iter_my_playlists())
|
||||||
@@ -215,15 +358,17 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
|||||||
Playlist.source == "local",
|
Playlist.source == "local",
|
||||||
Playlist.yt_playlist_id.is_not(None),
|
Playlist.yt_playlist_id.is_not(None),
|
||||||
)
|
)
|
||||||
)
|
).scalars()
|
||||||
.scalars()
|
|
||||||
.all()
|
|
||||||
)
|
)
|
||||||
# Drop mirrors whose YouTube playlist no longer exists.
|
# Drop vanished mirrors (none on an empty answer) — locked first, as every item write.
|
||||||
for ytid, pl in existing.items():
|
gone = vanished(existing, yt_ids)
|
||||||
if ytid not in yt_ids:
|
if gone:
|
||||||
db.delete(pl)
|
lock_playlists(db, *(pl.id for pl in gone))
|
||||||
db.flush()
|
for pl in gone:
|
||||||
|
db.delete(pl)
|
||||||
|
# Committed now: a run where no list below commits (all dirty, or all claimed) would
|
||||||
|
# otherwise drop the deletions with the session.
|
||||||
|
db.commit()
|
||||||
|
|
||||||
for idx, p in enumerate(yt_playlists):
|
for idx, p in enumerate(yt_playlists):
|
||||||
ytid = p.get("id")
|
ytid = p.get("id")
|
||||||
@@ -245,12 +390,30 @@ def sync_user_playlists(db: Session, user: User) -> dict:
|
|||||||
)
|
)
|
||||||
db.add(pl)
|
db.add(pl)
|
||||||
db.flush()
|
db.flush()
|
||||||
else:
|
with sync_claim(db, pl.id) as free:
|
||||||
|
if not free:
|
||||||
|
# The user is pushing or re-pulling this very list: it writes the same rows.
|
||||||
|
# The next scheduled run refreshes it.
|
||||||
|
continue
|
||||||
|
ids = _mirror_listing(yt, ytid, p.get("item_count"), pl.id)
|
||||||
|
if ids is None:
|
||||||
|
db.rollback() # a mirror this run just created is not kept empty either
|
||||||
|
continue
|
||||||
|
# Ingest first — it commits — so the row lock below is held from the dirty check
|
||||||
|
# to the commit that replaces the items.
|
||||||
|
_ensure_videos(db, yt, ids)
|
||||||
|
# `dirty` read above is a snapshot from before the YouTube calls (sessions keep
|
||||||
|
# loaded rows past a commit): a remove or reorder made meanwhile must not be
|
||||||
|
# replaced by YouTube's order. Item writes lock the playlist row, so this waits for
|
||||||
|
# one in flight and sees what it committed.
|
||||||
|
lock_playlists(db, pl.id)
|
||||||
|
if pl.dirty:
|
||||||
|
db.rollback()
|
||||||
|
continue
|
||||||
pl.name = p.get("title") or pl.name
|
pl.name = p.get("title") or pl.name
|
||||||
# Mirror is authoritative: replace the items to match YouTube's order.
|
# Mirror is authoritative: replace the items to match YouTube's order.
|
||||||
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
_replace_items_from_youtube(db, pl, ids)
|
||||||
_replace_items_from_youtube(db, yt, pl, ids)
|
db.commit()
|
||||||
db.commit()
|
|
||||||
synced += 1
|
synced += 1
|
||||||
return {"synced": synced}
|
return {"synced": synced}
|
||||||
|
|
||||||
@@ -267,90 +430,6 @@ def _local_video_ids(db: Session, pl: Playlist) -> list[str]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _reorder_moves(current: list[str], desired: list[str]) -> int:
|
|
||||||
"""Count the playlistItems.update moves an insertion-sort would need to turn
|
|
||||||
`current` order into `desired` order (both already the same set of video ids).
|
|
||||||
Items already in place are skipped; this matches what apply_push actually executes."""
|
|
||||||
model = list(current)
|
|
||||||
moves = 0
|
|
||||||
for i, want in enumerate(desired):
|
|
||||||
if i < len(model) and model[i] == want:
|
|
||||||
continue
|
|
||||||
j = model.index(want, i)
|
|
||||||
model.pop(j)
|
|
||||||
model.insert(i, want)
|
|
||||||
moves += 1
|
|
||||||
return moves
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class MergePlan:
|
|
||||||
"""The 3-way merge of a linked playlist: what each side gains and loses so neither side
|
|
||||||
loses an item, plus the merged order. `final` is the id set both sides end up with."""
|
|
||||||
|
|
||||||
final: list[str] # merged order (local order, YouTube-only adds appended)
|
|
||||||
add_to_yt: list[str] # local-adds to push out to YouTube
|
|
||||||
add_to_local: list[str] # YouTube-adds to pull into Siftlode
|
|
||||||
remove_from_yt: list[str] # items deleted locally that are still on YouTube
|
|
||||||
remove_from_local: list[str] # items deleted on YouTube that are still local
|
|
||||||
reorders: int # estimated YouTube playlistItems.update moves to reach `final`
|
|
||||||
baseline_known: bool
|
|
||||||
|
|
||||||
|
|
||||||
def merge_plan(
|
|
||||||
baseline: list[str] | None,
|
|
||||||
local: list[str],
|
|
||||||
youtube: list[str],
|
|
||||||
unavailable: set[str] | None = None,
|
|
||||||
) -> MergePlan:
|
|
||||||
"""Pure 3-way merge. Classify each id against the last-synced `baseline`:
|
|
||||||
- in local but not youtube, not deleted → add to YouTube (a local-add)
|
|
||||||
- in youtube but not local, not deleted → add to Siftlode (a YouTube-add)
|
|
||||||
- in baseline but gone from local → a genuine local delete → propagate to YouTube
|
|
||||||
- in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode
|
|
||||||
With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both
|
|
||||||
sides (never delete) — the lossless default. Merged order: local order first, YouTube-only
|
|
||||||
additions appended in their YouTube order.
|
|
||||||
`unavailable` = known un-ingestable-but-live YouTube ids: dropped from the YouTube side up
|
|
||||||
front so they're never re-classified as a YouTube-add (they can't ingest) nor as a local
|
|
||||||
delete (they were never local) — otherwise such a video would make the playlist never
|
|
||||||
converge. They stay in place on YouTube, untouched by this merge."""
|
|
||||||
un = set(unavailable or ())
|
|
||||||
L = list(dict.fromkeys(local))
|
|
||||||
Y = [v for v in dict.fromkeys(youtube) if v not in un]
|
|
||||||
Lset, Yset = set(L), set(Y)
|
|
||||||
if baseline is None:
|
|
||||||
add_to_yt = [v for v in L if v not in Yset]
|
|
||||||
add_to_local = [v for v in Y if v not in Lset]
|
|
||||||
remove_from_yt: list[str] = []
|
|
||||||
remove_from_local: list[str] = []
|
|
||||||
final = L + [v for v in Y if v not in Lset]
|
|
||||||
else:
|
|
||||||
B = set(baseline)
|
|
||||||
deleted_local = B - Lset # was synced, removed on our side
|
|
||||||
deleted_yt = B - Yset # was synced, removed on YouTube's side
|
|
||||||
gone = deleted_local | deleted_yt
|
|
||||||
remove_from_yt = [v for v in Y if v in deleted_local]
|
|
||||||
remove_from_local = [v for v in L if v in deleted_yt]
|
|
||||||
add_to_yt = [v for v in L if v not in Yset and v not in gone]
|
|
||||||
add_to_local = [v for v in Y if v not in Lset and v not in gone]
|
|
||||||
final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone]
|
|
||||||
final_set = set(final)
|
|
||||||
# On YouTube, after membership reconciliation the order is current-kept + inserts appended;
|
|
||||||
# the reorder pass then moves items into `final`.
|
|
||||||
after_membership = [v for v in Y if v in final_set] + add_to_yt
|
|
||||||
reorders = _reorder_moves(after_membership, final)
|
|
||||||
return MergePlan(
|
|
||||||
final=final,
|
|
||||||
add_to_yt=add_to_yt,
|
|
||||||
add_to_local=add_to_local,
|
|
||||||
remove_from_yt=remove_from_yt,
|
|
||||||
remove_from_local=remove_from_local,
|
|
||||||
reorders=reorders,
|
|
||||||
baseline_known=baseline is not None,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||||
"""Compute (without writing) what a two-way sync of this playlist would do: create vs
|
"""Compute (without writing) what a two-way sync of this playlist would do: create vs
|
||||||
update, how many items each side gains/loses, reorders, and the quota estimate (YouTube
|
update, how many items each side gains/loses, reorders, and the quota estimate (YouTube
|
||||||
@@ -378,6 +457,7 @@ def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|||||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||||
rename = bool(snippet) and (snippet.get("title") or "") != pl.name
|
rename = bool(snippet) and (snippet.get("title") or "") != pl.name
|
||||||
yt_ids = [it["video_id"] for it in current]
|
yt_ids = [it["video_id"] for it in current]
|
||||||
|
refuse_blank_listing(yt_ids, (snippet or {}).get("item_count"))
|
||||||
mp = merge_plan(parse_baseline(pl), local, yt_ids, parse_unavailable(pl))
|
mp = merge_plan(parse_baseline(pl), local, yt_ids, parse_unavailable(pl))
|
||||||
ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0)
|
ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0)
|
||||||
return {
|
return {
|
||||||
@@ -488,7 +568,8 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|||||||
breaker = _Breaker(playlist_id=pl.id)
|
breaker = _Breaker(playlist_id=pl.id)
|
||||||
with YouTubeClient(db, user) as yt:
|
with YouTubeClient(db, user) as yt:
|
||||||
if not pl.yt_playlist_id:
|
if not pl.yt_playlist_id:
|
||||||
pl.yt_playlist_id = yt.create_playlist(pl.name)
|
pushed_name = pl.name
|
||||||
|
pl.yt_playlist_id = yt.create_playlist(pushed_name)
|
||||||
db.commit()
|
db.commit()
|
||||||
created = 1
|
created = 1
|
||||||
# Brand-new playlist: just append every local video in order.
|
# Brand-new playlist: just append every local video in order.
|
||||||
@@ -501,7 +582,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|||||||
if not isinstance(added, _Failed):
|
if not isinstance(added, _Failed):
|
||||||
inserted += 1
|
inserted += 1
|
||||||
if not failures:
|
if not failures:
|
||||||
_set_baseline(pl, local)
|
_set_baseline_under_lock(db, pl, local, name=pushed_name)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {
|
return {
|
||||||
"created": created,
|
"created": created,
|
||||||
@@ -523,6 +604,8 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|||||||
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
||||||
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
||||||
yt_ids = [it["video_id"] for it in current]
|
yt_ids = [it["video_id"] for it in current]
|
||||||
|
# A merge would read "YouTube lists none of them" as "YouTube deleted them all".
|
||||||
|
refuse_blank_listing(yt_ids, (snippet or {}).get("item_count"))
|
||||||
# Re-attempt ingestion of ids previously marked un-ingestable that are still on YouTube: a
|
# Re-attempt ingestion of ids previously marked un-ingestable that are still on YouTube: a
|
||||||
# video that has since become available (e.g. private→public) should re-enter the merge, not
|
# video that has since become available (e.g. private→public) should re-enter the merge, not
|
||||||
# stay skipped forever. Those that still won't ingest remain filtered (`effective_unavailable`).
|
# stay skipped forever. Those that still won't ingest remain filtered (`effective_unavailable`).
|
||||||
@@ -542,6 +625,14 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|||||||
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
# merged order authoritatively. Do this first so local reflects the merge even if a later
|
||||||
# YouTube write trips the breaker.
|
# YouTube write trips the breaker.
|
||||||
_ensure_videos(db, yt, mp.add_to_local)
|
_ensure_videos(db, yt, mp.add_to_local)
|
||||||
|
# `local` was read before the YouTube calls above: an add or remove committed meanwhile
|
||||||
|
# (another tab) would be undone by the replace below. Lock the list — the ingest's commit is
|
||||||
|
# behind us — and plan again from what is there now. Nothing new needs ingesting: a video
|
||||||
|
# the re-read adds is already a local row. A list deleted meanwhile stops here, before a plan
|
||||||
|
# over its now-empty items removes them all from YouTube.
|
||||||
|
_lock_or_gone(db, pl)
|
||||||
|
local = _local_video_ids(db, pl)
|
||||||
|
mp = merge_plan(parse_baseline(pl), local, yt_ids, effective_unavailable)
|
||||||
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
|
# Only the ids that actually exist in the catalog can become local rows (a private/deleted
|
||||||
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
|
# YouTube video won't ingest); keep the merged order, dropping any that didn't materialize.
|
||||||
present = (
|
present = (
|
||||||
@@ -555,10 +646,12 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|||||||
_set_local_items(db, pl, final_local)
|
_set_local_items(db, pl, final_local)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
# --- YouTube side (metered, breaker-guarded).
|
# --- YouTube side (metered, breaker-guarded). The name as of the lock above: the closing
|
||||||
if snippet is not None and (snippet.get("title") or "") != pl.name:
|
# baseline records THIS one, whatever a rename made during the writes below says.
|
||||||
|
pushed_name = pl.name
|
||||||
|
if snippet is not None and (snippet.get("title") or "") != pushed_name:
|
||||||
try:
|
try:
|
||||||
yt.update_playlist_title(pl.yt_playlist_id, pl.name)
|
yt.update_playlist_title(pl.yt_playlist_id, pushed_name)
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
# All the caller passes on is the word "title", which says the rename did not
|
# All the caller passes on is the word "title", which says the rename did not
|
||||||
# happen but not that YouTube refused it, or why.
|
# happen but not that YouTube refused it, or why.
|
||||||
@@ -626,7 +719,7 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|||||||
(effective_unavailable | {v for v in mp.add_to_local if v not in present})
|
(effective_unavailable | {v for v in mp.add_to_local if v not in present})
|
||||||
& yt_id_set
|
& yt_id_set
|
||||||
)
|
)
|
||||||
_set_baseline(pl, final_local, unavailable)
|
_set_baseline_under_lock(db, pl, final_local, unavailable, name=pushed_name)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {
|
return {
|
||||||
"created": created,
|
"created": created,
|
||||||
@@ -651,10 +744,15 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
|||||||
with YouTubeClient(db, user) as yt:
|
with YouTubeClient(db, user) as yt:
|
||||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||||
if snippet is None:
|
if snippet is None:
|
||||||
raise YouTubeError("Playlist no longer exists on YouTube")
|
raise YouTubeError(
|
||||||
pl.name = snippet.get("title") or pl.name
|
"Playlist no longer exists on YouTube", reason="playlistNotFound", path="playlists"
|
||||||
|
)
|
||||||
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
||||||
ordered = _replace_items_from_youtube(db, yt, pl, ids)
|
refuse_blank_listing(ids, snippet.get("item_count"))
|
||||||
|
_ensure_videos(db, yt, ids)
|
||||||
|
_lock_or_gone(db, pl)
|
||||||
|
pl.name = snippet.get("title") or pl.name
|
||||||
|
ordered = _replace_items_from_youtube(db, pl, ids)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"items": len(ordered)}
|
return {"items": len(ordered)}
|
||||||
|
|
||||||
|
|||||||
@@ -15,8 +15,6 @@ from sqlalchemy import func, or_, select, update
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs, progress, quota, sysconfig
|
from app import logs, progress, quota, sysconfig
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.models import Channel, OAuthToken, Subscription, User
|
from app.models import Channel, OAuthToken, Subscription, User
|
||||||
from app.sync.explore import followed_channel
|
from app.sync.explore import followed_channel
|
||||||
@@ -35,6 +33,8 @@ from app.sync.videos import (
|
|||||||
from app.youtube.client import YouTubeClient, YouTubeError, is_quota_refusal
|
from app.youtube.client import YouTubeClient, YouTubeError, is_quota_refusal
|
||||||
from app.youtube.rss import RssError, fetch_channel_feed
|
from app.youtube.rss import RssError, fetch_channel_feed
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
|
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
|
||||||
RSS_POLL_WORKERS = 16
|
RSS_POLL_WORKERS = 16
|
||||||
# Failed channels named in a poll's summary warning; the rest are counted by reason.
|
# Failed channels named in a poll's summary warning; the rest are counted by reason.
|
||||||
@@ -83,6 +83,8 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
|||||||
# outage from one permanently dead feed id — which is the whole difference on an instance with
|
# outage from one permanently dead feed id — which is the whole difference on an instance with
|
||||||
# a single subscription, where BOTH render as 100% failed.
|
# a single subscription, where BOTH render as 100% failed.
|
||||||
first_error: str | None = None
|
first_error: str | None = None
|
||||||
|
# The same first failure as data, for the reader's language: whose feed, and why (`reasons`' key).
|
||||||
|
first_failure: tuple[str, str] | None = None
|
||||||
# What failed, as counts per reason (`http_404`, `transport`, `apply`, …) plus a few examples —
|
# What failed, as counts per reason (`http_404`, `transport`, `apply`, …) plus a few examples —
|
||||||
# the ONE warning a poll writes. One line per failed channel was what turned YouTube's RSS outage
|
# the ONE warning a poll writes. One line per failed channel was what turned YouTube's RSS outage
|
||||||
# of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now.
|
# of 2026-09-16 into 11k warnings in four hours; each channel's own line is DEBUG now.
|
||||||
@@ -92,18 +94,19 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
|||||||
# hand the same rows to the same broken write) — the feed fallback's candidates.
|
# hand the same rows to the same broken write) — the feed fallback's candidates.
|
||||||
unread: list[str] = []
|
unread: list[str] = []
|
||||||
|
|
||||||
def fail(reason: str, detail: str, *, unexpected: str | None = None) -> None:
|
def fail(reason: str, channel_id: str, detail: str, *, unexpected: str | None = None) -> None:
|
||||||
"""Count a channel that did not get polled. `unexpected` is the message for a failure that
|
"""Count a channel that did not get polled. `unexpected` is the message for a failure that
|
||||||
is a bug on our side: its FIRST occurrence per reason gets a traceback at ERROR, the rest
|
is a bug on our side: its FIRST occurrence per reason gets a traceback at ERROR, the rest
|
||||||
DEBUG — a read-only database fails every channel the same way, and 979 identical tracebacks
|
DEBUG — a read-only database fails every channel the same way, and 979 identical tracebacks
|
||||||
tell nobody anything the first one didn't. Call it from inside the `except` block."""
|
tell nobody anything the first one didn't. Call it from inside the `except` block."""
|
||||||
nonlocal failed, first_error
|
nonlocal failed, first_error, first_failure
|
||||||
if unexpected is not None:
|
if unexpected is not None:
|
||||||
level = logging.ERROR if reasons[reason] == 0 else logging.DEBUG
|
level = logging.ERROR if reasons[reason] == 0 else logging.DEBUG
|
||||||
log.log(level, unexpected, exc_info=True)
|
log.log(level, unexpected, exc_info=True)
|
||||||
failed += 1
|
failed += 1
|
||||||
reasons[reason] += 1
|
reasons[reason] += 1
|
||||||
first_error = first_error or detail
|
first_error = first_error or detail
|
||||||
|
first_failure = first_failure or (channel_id, reason)
|
||||||
if len(examples) < RSS_FAILURE_EXAMPLES:
|
if len(examples) < RSS_FAILURE_EXAMPLES:
|
||||||
examples.append(detail)
|
examples.append(detail)
|
||||||
|
|
||||||
@@ -125,13 +128,16 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
|||||||
# bug on our side and earns a traceback (the first of its kind per poll).
|
# bug on our side and earns a traceback (the first of its kind per poll).
|
||||||
except RssError as exc:
|
except RssError as exc:
|
||||||
log.debug("RSS poll skipped for channel %s: %s", channel.id, exc)
|
log.debug("RSS poll skipped for channel %s: %s", channel.id, exc)
|
||||||
fail(exc.reason, f"{channel.id}: {exc}")
|
fail(exc.reason, channel.id, f"{channel.id}: {exc}")
|
||||||
unread.append(channel.id)
|
unread.append(channel.id)
|
||||||
# silent-ok: `fail(unexpected=…)` is the report — it logs at ERROR with the
|
# silent-ok: `fail(unexpected=…)` is the report — it logs at ERROR with the
|
||||||
# traceback, once per reason per poll, which is the flood control this loop needs.
|
# traceback, once per reason per poll, which is the flood control this loop needs.
|
||||||
except Exception as exc: # noqa: BLE001 — one channel must not stop the poll
|
except Exception as exc: # noqa: BLE001 — one channel must not stop the poll
|
||||||
fail(
|
fail(
|
||||||
"error", f"{channel.id}: {exc}", unexpected=f"RSS fetch failed for {channel.id}"
|
"error",
|
||||||
|
channel.id,
|
||||||
|
f"{channel.id}: {exc}",
|
||||||
|
unexpected=f"RSS fetch failed for {channel.id}",
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
try:
|
try:
|
||||||
@@ -144,6 +150,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
|||||||
db.rollback()
|
db.rollback()
|
||||||
fail(
|
fail(
|
||||||
"apply",
|
"apply",
|
||||||
|
channel.id,
|
||||||
f"{channel.id} (apply): {exc}",
|
f"{channel.id} (apply): {exc}",
|
||||||
unexpected=f"RSS apply failed for {channel.id}",
|
unexpected=f"RSS apply failed for {channel.id}",
|
||||||
)
|
)
|
||||||
@@ -180,9 +187,17 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
|||||||
# "fix this subscription"; `Connection refused` says "the egress is down".
|
# "fix this subscription"; `Connection refused` says "the egress is down".
|
||||||
# The fallback's count rides along: a red card during an RSS outage should still say
|
# The fallback's count rides along: a red card during an RSS outage should still say
|
||||||
# whether the uploads are arriving another way.
|
# whether the uploads are arriving another way.
|
||||||
raise RuntimeError(
|
# As data too (E31): the audit log and the inbox say it in the reader's language, the first
|
||||||
|
# failure as its channel and its reason code (`http_404` vs `transport` is that difference).
|
||||||
|
first_channel, first_reason = first_failure or ("", "error")
|
||||||
|
raise progress.JobFailed(
|
||||||
|
"rss_all_failed",
|
||||||
f"RSS poll failed for all {total} channels — first: {first_error}"
|
f"RSS poll failed for all {total} channels — first: {first_error}"
|
||||||
f" (read through the API instead: {fallback['read']})"
|
f" (read through the API instead: {fallback['read']})",
|
||||||
|
total=total,
|
||||||
|
channel=first_channel,
|
||||||
|
cause=first_reason,
|
||||||
|
read=fallback["read"],
|
||||||
)
|
)
|
||||||
return {"new": new + fallback["new"], "failed": failed, "fallback": fallback["read"]}
|
return {"new": new + fallback["new"], "failed": failed, "fallback": fallback["read"]}
|
||||||
|
|
||||||
@@ -252,7 +267,7 @@ def run_feed_fallback(db: Session, channel_ids: list[str]) -> dict[str, int]:
|
|||||||
channel.feed_fallback_at = datetime.now(timezone.utc)
|
channel.feed_fallback_at = datetime.now(timezone.utc)
|
||||||
db.add(channel)
|
db.add(channel)
|
||||||
db.commit()
|
db.commit()
|
||||||
except Exception: # noqa: BLE001 — one channel must not stop the rest
|
except Exception: # one channel must not stop the rest
|
||||||
db.rollback()
|
db.rollback()
|
||||||
log.exception(
|
log.exception(
|
||||||
"Feed fallback could not store channel %s",
|
"Feed fallback could not store channel %s",
|
||||||
|
|||||||
@@ -1,14 +1,15 @@
|
|||||||
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
from collections.abc import Collection, Iterable
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs
|
from app import logs
|
||||||
from app.models import Channel, Subscription, User
|
from app.models import Channel, Subscription, User
|
||||||
from app.utils import normalize_description
|
from app.utils import normalize_description
|
||||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
@@ -47,8 +48,37 @@ def _external_links(branding: dict) -> list | None:
|
|||||||
return out or None
|
return out or None
|
||||||
|
|
||||||
|
|
||||||
def apply_channel_details(db: Session, items: list[dict]) -> None:
|
# How old a channel's metadata may get before it is fetched again. Banners, avatars, descriptions and
|
||||||
|
# counts change on YouTube, and an old banner URL stops resolving at all — without a refresh a channel
|
||||||
|
# kept whatever it had the day it was first seen. A week keeps counts roughly current at almost no
|
||||||
|
# cost: `channels.list` is 1 quota unit per 50 channels, and only the stale ones are asked for.
|
||||||
|
DETAILS_MAX_AGE = timedelta(days=7)
|
||||||
|
|
||||||
|
|
||||||
|
def details_stale(channel: Channel, now: datetime) -> bool:
|
||||||
|
"""Whether `channel`'s metadata is missing or older than DETAILS_MAX_AGE (see `details_stale_clause`)."""
|
||||||
|
synced = channel.details_synced_at
|
||||||
|
return synced is None or synced < now - DETAILS_MAX_AGE
|
||||||
|
|
||||||
|
|
||||||
|
def details_stale_clause(now: datetime):
|
||||||
|
"""`details_stale` as a WHERE clause, so the batch import and a single channel agree on "stale"."""
|
||||||
|
return or_(
|
||||||
|
Channel.details_synced_at.is_(None),
|
||||||
|
Channel.details_synced_at < now - DETAILS_MAX_AGE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_channel_details(db: Session, items: list[dict], asked: Iterable[str]) -> None:
|
||||||
|
"""Write `channels.list`'s answer onto the rows. `asked` is every id the call asked for: one
|
||||||
|
YouTube did not return (terminated, deleted) is stamped as synced too — otherwise it stays
|
||||||
|
stale forever and every view / import asks for it again, spending quota on a known miss."""
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
|
returned = {item["id"] for item in items}
|
||||||
|
for cid in set(asked) - returned:
|
||||||
|
missing = db.get(Channel, cid)
|
||||||
|
if missing is not None:
|
||||||
|
missing.details_synced_at = now
|
||||||
for item in items:
|
for item in items:
|
||||||
channel = db.get(Channel, item["id"])
|
channel = db.get(Channel, item["id"])
|
||||||
if channel is None:
|
if channel is None:
|
||||||
@@ -85,14 +115,41 @@ def should_prune(fetched_count: int, existing_count: int) -> bool:
|
|||||||
has — taking each row's `deep_requested` flag with it — in a single run. Someone who genuinely
|
has — taking each row's `deep_requested` flag with it — in a single run. Someone who genuinely
|
||||||
unsubscribed from everything simply keeps their rows until a sync that returns something.
|
unsubscribed from everything simply keeps their rows until a sync that returns something.
|
||||||
|
|
||||||
Pure so it can be tested without a database; `import_subscriptions` is the only caller."""
|
Pure so it can be tested without a database; its callers are `import_subscriptions` and, through
|
||||||
|
`vanished`, every mirror that drops what the remote no longer lists."""
|
||||||
return fetched_count > 0 or existing_count == 0
|
return fetched_count > 0 or existing_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
def vanished[K, V](existing: dict[K, V], listed: Collection[K]) -> list[V]:
|
||||||
|
"""The local mirrors whose key the remote listing no longer has — none at all when the listing
|
||||||
|
is an empty answer that contradicts them (`should_prune`). The YouTube playlist mirrors use it:
|
||||||
|
without it, one blip dropped every mirror (E32 S2)."""
|
||||||
|
if not should_prune(len(listed), len(existing)):
|
||||||
|
return []
|
||||||
|
return [row for key, row in existing.items() if key not in listed]
|
||||||
|
|
||||||
|
|
||||||
|
def blank_listing(listed: Collection, count: object) -> bool:
|
||||||
|
"""The same rule one level down: no members from a container whose own listing counts some (a
|
||||||
|
playlist's `itemCount`) is a blip, not a list emptied. Replacing or merging on it deleted every
|
||||||
|
mirrored item and reset the merge baseline (E32 review)."""
|
||||||
|
return not listed and isinstance(count, int) and count > 0
|
||||||
|
|
||||||
|
|
||||||
|
def refuse_blank_listing(listed: Collection, count: object) -> None:
|
||||||
|
"""`blank_listing` for a user-started pull or push: refused as retryable ("try again")."""
|
||||||
|
if blank_listing(listed, count):
|
||||||
|
raise YouTubeError(
|
||||||
|
"YouTube listed no items for a playlist it counts as non-empty",
|
||||||
|
reason="emptyAnswer",
|
||||||
|
path="playlistItems",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def import_subscriptions(db: Session, user: User) -> dict:
|
def import_subscriptions(db: Session, user: User) -> dict:
|
||||||
"""Pull all subscriptions for `user`, upsert channels + subscription links, prune
|
"""Pull all subscriptions for `user`, upsert channels + subscription links, prune
|
||||||
channels the user has unsubscribed from on YouTube, and fetch channel details for
|
channels the user has unsubscribed from on YouTube, and fetch channel details for
|
||||||
any channel we don't have details for yet."""
|
any channel whose details are missing or stale (`DETAILS_MAX_AGE`)."""
|
||||||
fetched: dict[str, dict] = {}
|
fetched: dict[str, dict] = {}
|
||||||
new_channels = 0
|
new_channels = 0
|
||||||
|
|
||||||
@@ -145,12 +202,12 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
|||||||
removed += 1
|
removed += 1
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
# Fetch details for channels that don't have them yet.
|
# Fetch details for channels that don't have them yet, or had them too long ago.
|
||||||
need_details = (
|
need_details = (
|
||||||
db.execute(
|
db.execute(
|
||||||
select(Channel).where(
|
select(Channel).where(
|
||||||
Channel.id.in_(fetched.keys()),
|
Channel.id.in_(fetched.keys()),
|
||||||
Channel.details_synced_at.is_(None),
|
details_stale_clause(datetime.now(timezone.utc)),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
.scalars()
|
.scalars()
|
||||||
@@ -158,8 +215,9 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
|||||||
)
|
)
|
||||||
detailed = 0
|
detailed = 0
|
||||||
if need_details:
|
if need_details:
|
||||||
items = yt.get_channels([c.id for c in need_details])
|
ids = [c.id for c in need_details]
|
||||||
apply_channel_details(db, items)
|
items = yt.get_channels(ids)
|
||||||
|
apply_channel_details(db, items, ids)
|
||||||
detailed = len(items)
|
detailed = len(items)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
|
|||||||
@@ -11,12 +11,14 @@ from sqlalchemy import and_, func, or_, select, update
|
|||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import progress, quota, sysconfig
|
from app import logs, progress, quota, sysconfig
|
||||||
from app.models import LIVE_OR_UPCOMING, Channel, Video
|
from app.models import LIVE_OR_UPCOMING, Channel, Video
|
||||||
from app.titles import normalize_title
|
from app.titles import normalize_title
|
||||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail, is_playlist_gone
|
||||||
from app.youtube.shorts import make_client, probe_is_short
|
from app.youtube.shorts import make_client, probe_is_short
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
_DURATION_RE = re.compile(r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
|
_DURATION_RE = re.compile(r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
@@ -104,6 +106,20 @@ def stub_from_playlist_item(item: dict, channel_id: str) -> dict | None:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _uploads_page(yt: YouTubeClient, channel_id: str, uploads: str, page_token: str | None) -> dict:
|
||||||
|
"""One page of the channel's uploads. YouTube answers the uploads list of a channel with no
|
||||||
|
uploads at all as not found: an empty back-catalogue, so an empty last page — not a failure
|
||||||
|
that left `recent_synced_at` unstamped and charged the channel again on every run (E32 review).
|
||||||
|
Any other failure raises."""
|
||||||
|
try:
|
||||||
|
return yt.get_playlist_items(uploads, page_token)
|
||||||
|
except YouTubeError as exc:
|
||||||
|
if not is_playlist_gone(exc):
|
||||||
|
raise
|
||||||
|
log.info("backfill channel=%s has no uploads playlist", channel_id)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> int:
|
def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> int:
|
||||||
"""First pass: newest videos up to the configured count or age cutoff. Records a
|
"""First pass: newest videos up to the configured count or age cutoff. Records a
|
||||||
resume cursor for the later deep backfill."""
|
resume cursor for the later deep backfill."""
|
||||||
@@ -121,7 +137,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
|
|||||||
|
|
||||||
while True:
|
while True:
|
||||||
page_start = page_token # token that fetched THIS page (None = first page)
|
page_start = page_token # token that fetched THIS page (None = first page)
|
||||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token)
|
||||||
stopped_mid_page = False
|
stopped_mid_page = False
|
||||||
for item in data.get("items", []):
|
for item in data.get("items", []):
|
||||||
stub = stub_from_playlist_item(item, channel.id)
|
stub = stub_from_playlist_item(item, channel.id)
|
||||||
@@ -175,7 +191,7 @@ def backfill_channel_deep(
|
|||||||
for _ in range(max_pages):
|
for _ in range(max_pages):
|
||||||
if quota.backfill_reserve_reached():
|
if quota.backfill_reserve_reached():
|
||||||
break
|
break
|
||||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token)
|
||||||
rows = [
|
rows = [
|
||||||
s
|
s
|
||||||
for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", []))
|
for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", []))
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ stored in the DB and stay env-only).
|
|||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs, security
|
from app import logs, security
|
||||||
@@ -45,8 +46,8 @@ class ConfigSpec:
|
|||||||
|
|
||||||
|
|
||||||
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
|
# Registry of DB-overridable keys. Add a key here + wire its read through get_*() to move it
|
||||||
# off env into the admin UI; the admin page renders it automatically from this list. The ORDER is the
|
# off env into the admin UI; the admin page renders it automatically from this list. A group's keys
|
||||||
# admin page's tab order (first appearance of each group), so keep a group's keys together.
|
# stay together, in the order its section lists them; a NEW group also needs its place in SECTIONS.
|
||||||
SPECS: tuple[ConfigSpec, ...] = (
|
SPECS: tuple[ConfigSpec, ...] = (
|
||||||
# --- Access / registration ---
|
# --- Access / registration ---
|
||||||
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
|
ConfigSpec("allow_registration", "bool", "access", "allow_registration"),
|
||||||
@@ -219,6 +220,16 @@ SPECS: tuple[ConfigSpec, ...] = (
|
|||||||
# is a config.py default — both stay out of SPECS, like plex_sync_interval_min.)
|
# is a config.py default — both stay out of SPECS, like plex_sync_interval_min.)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# The admin page's sections (E31, the owner's pick): sixteen groups as tabs wrapped to three rows at 133%
|
||||||
|
# zoom, so they are a vertical list under four headings, in this order. Every group of SPECS sits in
|
||||||
|
# exactly one section (test-held), so a new group cannot fall off the page.
|
||||||
|
SECTIONS: tuple[tuple[str, tuple[str, ...]], ...] = (
|
||||||
|
("access", ("access", "google", "email")),
|
||||||
|
("youtube", ("youtube", "quota", "backfill", "shorts", "batch", "explore")),
|
||||||
|
("media", ("downloads", "plex", "tmdb")),
|
||||||
|
("system", ("maintenance", "audit", "diagnostics", "issues")),
|
||||||
|
)
|
||||||
|
|
||||||
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}
|
_BY_KEY: dict[str, ConfigSpec] = {s.key: s for s in SPECS}
|
||||||
|
|
||||||
|
|
||||||
@@ -297,7 +308,11 @@ def _raw_override(db: Session, key: str) -> str | None:
|
|||||||
per key so the operator gets a real signal without a per-request traceback. Recovery is documented in
|
per key so the operator gets a real signal without a per-request traceback. Recovery is documented in
|
||||||
docs/self-hosting.md: the key must never change once secrets are stored. (B-core-2)
|
docs/self-hosting.md: the key must never change once secrets are stored. (B-core-2)
|
||||||
"""
|
"""
|
||||||
row = db.get(SystemConfig, key)
|
return _raw_of(db.get(SystemConfig, key), key)
|
||||||
|
|
||||||
|
|
||||||
|
def _raw_of(row: SystemConfig | None, key: str) -> str | None:
|
||||||
|
"""`_raw_override` over a row already read (describe() reads them all in one query)."""
|
||||||
if row is None or row.value is None:
|
if row is None or row.value is None:
|
||||||
return None
|
return None
|
||||||
if not row.encrypted:
|
if not row.encrypted:
|
||||||
@@ -317,8 +332,11 @@ def _raw_override(db: Session, key: str) -> str | None:
|
|||||||
|
|
||||||
def get(db: Session, key: str):
|
def get(db: Session, key: str):
|
||||||
"""Effective value: the DB override (typed) if set, else the env/config default."""
|
"""Effective value: the DB override (typed) if set, else the env/config default."""
|
||||||
s = _BY_KEY[key]
|
return _effective(_BY_KEY[key], _raw_override(db, key))
|
||||||
raw = _raw_override(db, key)
|
|
||||||
|
|
||||||
|
def _effective(s: ConfigSpec, raw: str | None):
|
||||||
|
"""`get` over an override already read."""
|
||||||
if raw is None:
|
if raw is None:
|
||||||
return _default(s)
|
return _default(s)
|
||||||
if raw == "" and not s.allow_empty:
|
if raw == "" and not s.allow_empty:
|
||||||
@@ -410,8 +428,12 @@ def describe(db: Session) -> dict:
|
|||||||
"""Admin-UI metadata + current values, grouped. Secret values are never echoed —
|
"""Admin-UI metadata + current values, grouped. Secret values are never echoed —
|
||||||
only whether an override is set and whether an env default exists."""
|
only whether an override is set and whether an env default exists."""
|
||||||
groups: dict[str, list[dict]] = {}
|
groups: dict[str, list[dict]] = {}
|
||||||
|
# Every override in one query: a db.get per key was one round trip each (two for a key with no
|
||||||
|
# row, which the session cannot cache), ~40 ms of the page's ~55 on prod (E31 S4d).
|
||||||
|
rows = {r.key: r for r in db.scalars(select(SystemConfig))}
|
||||||
for s in SPECS:
|
for s in SPECS:
|
||||||
is_set = db.get(SystemConfig, s.key) is not None
|
row = rows.get(s.key)
|
||||||
|
is_set = row is not None
|
||||||
item: dict = {
|
item: dict = {
|
||||||
"key": s.key,
|
"key": s.key,
|
||||||
"type": s.type,
|
"type": s.type,
|
||||||
@@ -426,7 +448,10 @@ def describe(db: Session) -> dict:
|
|||||||
if s.secret:
|
if s.secret:
|
||||||
item["default_is_set"] = bool(_default(s))
|
item["default_is_set"] = bool(_default(s))
|
||||||
else:
|
else:
|
||||||
item["value"] = get(db, s.key)
|
item["value"] = _effective(s, _raw_of(row, s.key))
|
||||||
item["default"] = _default(s)
|
item["default"] = _default(s)
|
||||||
groups.setdefault(s.group, []).append(item)
|
groups.setdefault(s.group, []).append(item)
|
||||||
return {"groups": groups, "secrets_manageable": secrets_manageable()}
|
sections = [
|
||||||
|
{"id": sid, "groups": [g for g in members if g in groups]} for sid, members in SECTIONS
|
||||||
|
]
|
||||||
|
return {"groups": groups, "sections": sections, "secrets_manageable": secrets_manageable()}
|
||||||
|
|||||||
@@ -60,7 +60,6 @@ _SHORT_LOWER = {
|
|||||||
"for",
|
"for",
|
||||||
"or",
|
"or",
|
||||||
"vs",
|
"vs",
|
||||||
"the",
|
|
||||||
"és",
|
"és",
|
||||||
"az",
|
"az",
|
||||||
"egy",
|
"egy",
|
||||||
@@ -120,7 +119,6 @@ _ACRONYMS = {
|
|||||||
"nfl",
|
"nfl",
|
||||||
"asmr",
|
"asmr",
|
||||||
"pov",
|
"pov",
|
||||||
"diy",
|
|
||||||
"wtf",
|
"wtf",
|
||||||
"lol",
|
"lol",
|
||||||
"rtx",
|
"rtx",
|
||||||
|
|||||||
+10
-2
@@ -60,16 +60,24 @@ def now_utc() -> datetime:
|
|||||||
return datetime.now(timezone.utc)
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
# The shortest last word that is prefix-matched. A 1–2 letter prefix matches a large share of the
|
||||||
|
# library and ts_rank then scores every match: "m" took ~5–8 s on prod against ~100 ms for a whole
|
||||||
|
# word (E31 S4e). A shorter last word is matched whole instead ("tv", "4k" still find themselves).
|
||||||
|
MIN_PREFIX_LEN = 3
|
||||||
|
|
||||||
|
|
||||||
def to_tsquery_str(q: str) -> str | None:
|
def to_tsquery_str(q: str) -> str | None:
|
||||||
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
|
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
|
||||||
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*".
|
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*" —
|
||||||
|
once it has MIN_PREFIX_LEN characters ("ti amo m" → "ti & amo & m").
|
||||||
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
|
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
|
||||||
(the string is built from sanitised tokens, never the raw query). Shared by the YouTube feed
|
(the string is built from sanitised tokens, never the raw query). Shared by the YouTube feed
|
||||||
search and the Plex library search, hence this neutral home rather than either domain module."""
|
search and the Plex library search, hence this neutral home rather than either domain module."""
|
||||||
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
|
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
|
||||||
if not tokens:
|
if not tokens:
|
||||||
return None
|
return None
|
||||||
tokens[-1] = tokens[-1] + ":*"
|
if len(tokens[-1]) >= MIN_PREFIX_LEN:
|
||||||
|
tokens[-1] = tokens[-1] + ":*"
|
||||||
return " & ".join(tokens)
|
return " & ".join(tokens)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
||||||
@@ -72,7 +73,7 @@ def _wait_for_schema(timeout: float = 300.0) -> bool:
|
|||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
|
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
|
||||||
return True
|
return True
|
||||||
except Exception: # noqa: BLE001 — table not migrated yet / DB not up yet
|
except Exception: # table not migrated yet / DB not up yet
|
||||||
if time.monotonic() > deadline:
|
if time.monotonic() > deadline:
|
||||||
# The traceback, not `str(exc)[:120]`: this is the last thing the worker says
|
# The traceback, not `str(exc)[:120]`: this is the last thing the worker says
|
||||||
# before it exits, and which driver call failed is the whole question.
|
# before it exits, and which driver call failed is the whole question.
|
||||||
@@ -366,7 +367,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
proc = _spawn_download_child(job_id)
|
proc = _spawn_download_child(job_id)
|
||||||
except Exception: # noqa: BLE001 — spawn failed; don't leave the asset stuck 'downloading'
|
except Exception: # spawn failed; don't leave the asset stuck 'downloading'
|
||||||
log.exception("failed to spawn download child for job %s", job_id)
|
log.exception("failed to spawn download child for job %s", job_id)
|
||||||
_reset_asset_pending(asset_id)
|
_reset_asset_pending(asset_id)
|
||||||
_requeue_if_running(job_id, None)
|
_requeue_if_running(job_id, None)
|
||||||
@@ -438,7 +439,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
|||||||
cause="child_exit",
|
cause="child_exit",
|
||||||
exit_code=proc.returncode,
|
exit_code=proc.returncode,
|
||||||
)
|
)
|
||||||
except Exception: # noqa: BLE001 — boot recovery is the backstop for a stuck 'running' job
|
except Exception: # boot recovery is the backstop for a stuck 'running' job
|
||||||
log.exception("post-exit reconciliation failed for job %s", job_id)
|
log.exception("post-exit reconciliation failed for job %s", job_id)
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -501,6 +502,7 @@ def _process_job(job_id: int) -> None:
|
|||||||
then run in a supervised CHILD PROCESS (`_run_supervised`) so a wedged yt-dlp/ffmpeg can be
|
then run in a supervised CHILD PROCESS (`_run_supervised`) so a wedged yt-dlp/ffmpeg can be
|
||||||
hard-killed — a worker THREAD cannot be killed, nor (when it spins in a C call) even
|
hard-killed — a worker THREAD cannot be killed, nor (when it spins in a C call) even
|
||||||
cooperatively cancelled, and one such thread starves the others via the GIL."""
|
cooperatively cancelled, and one such thread starves the others via the GIL."""
|
||||||
|
diagnosis = None # set exactly when the asset has already failed (asset_status == "error")
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
job = db.get(DownloadJob, job_id)
|
job = db.get(DownloadJob, job_id)
|
||||||
if job is None:
|
if job is None:
|
||||||
@@ -528,7 +530,7 @@ def _process_job(job_id: int) -> None:
|
|||||||
if asset_status == "ready":
|
if asset_status == "ready":
|
||||||
_finish_from_cache(job_id, asset_id)
|
_finish_from_cache(job_id, asset_id)
|
||||||
return
|
return
|
||||||
if asset_status == "error":
|
if diagnosis is not None:
|
||||||
_fail_running(
|
_fail_running(
|
||||||
job_id,
|
job_id,
|
||||||
error=asset_error or "Download failed",
|
error=asset_error or "Download failed",
|
||||||
@@ -577,7 +579,7 @@ def _worker_loop(idx: int) -> None:
|
|||||||
while not _stop.is_set():
|
while not _stop.is_set():
|
||||||
try:
|
try:
|
||||||
job_id = _claim_job()
|
job_id = _claim_job()
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
log.exception("claim failed")
|
log.exception("claim failed")
|
||||||
_stop.wait(settings.download_poll_seconds)
|
_stop.wait(settings.download_poll_seconds)
|
||||||
continue
|
continue
|
||||||
@@ -597,12 +599,12 @@ def _worker_loop(idx: int) -> None:
|
|||||||
job_id,
|
job_id,
|
||||||
exc,
|
exc,
|
||||||
"crash",
|
"crash",
|
||||||
lambda: _fail_running(job_id, error="Worker error", error_code=None),
|
partial(_fail_running, job_id, error="Worker error", error_code=None),
|
||||||
)
|
)
|
||||||
# silent-ok: not silent — `_report_worker_error` logged the crash before this
|
# silent-ok: not silent — `_report_worker_error` logged the crash before this
|
||||||
# re-raise. Failing the job hit the database, and letting that out would end this
|
# re-raise. Failing the job hit the database, and letting that out would end this
|
||||||
# worker thread for good; boot recovery settles a job left `running`.
|
# worker thread for good; boot recovery settles a job left `running`.
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
log.exception(
|
log.exception(
|
||||||
"could not fail job %s after its crash",
|
"could not fail job %s after its crash",
|
||||||
job_id,
|
job_id,
|
||||||
|
|||||||
@@ -114,18 +114,32 @@ _DEAD_GRANT = "invalid_grant"
|
|||||||
|
|
||||||
# What a refused YouTube action means to the person who asked for it — the three things they can
|
# What a refused YouTube action means to the person who asked for it — the three things they can
|
||||||
# act on differently: wait for tomorrow's budget, reconnect the account, or try again later.
|
# act on differently: wait for tomorrow's budget, reconnect the account, or try again later.
|
||||||
YouTubeRefusal = Literal["quota", "reconnect", "unavailable"]
|
YouTubeRefusal = Literal["quota", "reconnect", "unavailable", "gone"]
|
||||||
|
|
||||||
|
|
||||||
def refusal_reason(exc: YouTubeError) -> YouTubeRefusal:
|
def is_playlist_gone(exc: YouTubeError) -> bool:
|
||||||
|
"""Whether YouTube answered that the playlist read or written does not exist. Only the caller
|
||||||
|
knows which playlist that was: a user's list deleted on YouTube, or a channel's uploads list,
|
||||||
|
which YouTube also answers so for a channel with no uploads at all."""
|
||||||
|
return "playlistNotFound" in exc.reason.split(",")
|
||||||
|
|
||||||
|
|
||||||
|
def refusal_reason(exc: YouTubeError, *, playlist: bool = False) -> YouTubeRefusal:
|
||||||
"""Sort a failure by what the user can do about it. A dead grant (Google refused the refresh
|
"""Sort a failure by what the user can do about it. A dead grant (Google refused the refresh
|
||||||
with `invalid_grant`, which also drops the grant — see `_drop_dead_grant`) or no usable token at
|
with `invalid_grant`, which also drops the grant — see `_drop_dead_grant`) or no usable token at
|
||||||
all (raised before any endpoint answered, so with no path and no transport reason) means the
|
all (raised before any endpoint answered, so with no path and no transport reason) means the
|
||||||
account must be reconnected; retrying cannot help. Any other token-endpoint failure — a 5xx, a
|
account must be reconnected; retrying cannot help. Any other token-endpoint failure — a 5xx, a
|
||||||
429, a misconfigured client — leaves the grant as it was: reconnecting would not fix it, and
|
429, a misconfigured client — leaves the grant as it was: reconnecting would not fix it, and
|
||||||
the account would still read as connected, with no connect button to press."""
|
the account would still read as connected, with no connect button to press.
|
||||||
|
|
||||||
|
`playlist`: the action was on ONE playlist the user picked, so a missing playlist is that list,
|
||||||
|
gone (deleted on YouTube) — neither a retry nor a reconnect helps. Anywhere else the playlist
|
||||||
|
is one the user never named (a channel's uploads, one list of a bulk pull), and "this playlist
|
||||||
|
no longer exists" would describe nothing on their screen (E32 review round 2)."""
|
||||||
if is_quota_refusal(exc):
|
if is_quota_refusal(exc):
|
||||||
return "quota"
|
return "quota"
|
||||||
|
if playlist and is_playlist_gone(exc):
|
||||||
|
return "gone"
|
||||||
if exc.path == "oauth2/token":
|
if exc.path == "oauth2/token":
|
||||||
return "reconnect" if exc.reason == _DEAD_GRANT else "unavailable"
|
return "reconnect" if exc.reason == _DEAD_GRANT else "unavailable"
|
||||||
if not exc.path and exc.reason != "transport":
|
if not exc.path and exc.reason != "transport":
|
||||||
@@ -189,7 +203,7 @@ class YouTubeClient:
|
|||||||
return cached
|
return cached
|
||||||
try:
|
try:
|
||||||
refresh = decrypt(tok.refresh_token_enc)
|
refresh = decrypt(tok.refresh_token_enc)
|
||||||
except Exception as exc: # noqa: BLE001 — Fernet InvalidToken (TOKEN_ENCRYPTION_KEY changed)
|
except Exception as exc: # Fernet InvalidToken (TOKEN_ENCRYPTION_KEY changed)
|
||||||
# A key mismatch must degrade like any other token problem (caller handles YouTubeError),
|
# A key mismatch must degrade like any other token problem (caller handles YouTubeError),
|
||||||
# not surface as a raw 500 on every scheduler job / connect check. (B-core-2)
|
# not surface as a raw 500 on every scheduler job / connect check. (B-core-2)
|
||||||
raise YouTubeError("Stored OAuth token is unreadable — reconnect your account") from exc
|
raise YouTubeError("Stored OAuth token is unreadable — reconnect your account") from exc
|
||||||
@@ -423,16 +437,22 @@ class YouTubeClient:
|
|||||||
return resp.json() if resp.content else {}
|
return resp.json() if resp.content else {}
|
||||||
|
|
||||||
def get_playlist_snippet(self, playlist_id: str) -> dict | None:
|
def get_playlist_snippet(self, playlist_id: str) -> dict | None:
|
||||||
"""The snippet (title/description) of one playlist, or None if it no longer
|
"""The snippet (title/description) of one playlist plus its `item_count` (what an empty
|
||||||
exists. 1 unit (OAuth, so private playlists work)."""
|
item listing is checked against), or None if it no longer exists. 1 unit (OAuth, so
|
||||||
|
private playlists work)."""
|
||||||
data = self._get(
|
data = self._get(
|
||||||
"playlists",
|
"playlists",
|
||||||
{"part": "snippet", "id": playlist_id, "maxResults": 1},
|
{"part": "snippet,contentDetails", "id": playlist_id, "maxResults": 1},
|
||||||
cost=1,
|
cost=1,
|
||||||
allow_key=False,
|
allow_key=False,
|
||||||
)
|
)
|
||||||
items = data.get("items", [])
|
items = data.get("items", [])
|
||||||
return items[0].get("snippet") if items else None
|
if not items:
|
||||||
|
return None
|
||||||
|
return {
|
||||||
|
**(items[0].get("snippet") or {}),
|
||||||
|
"item_count": items[0].get("contentDetails", {}).get("itemCount"),
|
||||||
|
}
|
||||||
|
|
||||||
def update_playlist_title(self, playlist_id: str, title: str) -> None:
|
def update_playlist_title(self, playlist_id: str, title: str) -> None:
|
||||||
"""Rename a playlist on YouTube. 50 units."""
|
"""Rename a playlist on YouTube. 50 units."""
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ class RssError(Exception):
|
|||||||
def _get_feed(channel_id: str) -> bytes:
|
def _get_feed(channel_id: str) -> bytes:
|
||||||
"""The feed's body, after up to `1 + len(RETRY_DELAYS)` tries; the LAST failure is raised."""
|
"""The feed's body, after up to `1 + len(RETRY_DELAYS)` tries; the LAST failure is raised."""
|
||||||
url = RSS_URL.format(channel_id=channel_id)
|
url = RSS_URL.format(channel_id=channel_id)
|
||||||
|
error: RssError | None = None
|
||||||
for attempt in range(len(RETRY_DELAYS) + 1):
|
for attempt in range(len(RETRY_DELAYS) + 1):
|
||||||
if attempt:
|
if attempt:
|
||||||
time.sleep(RETRY_DELAYS[attempt - 1])
|
time.sleep(RETRY_DELAYS[attempt - 1])
|
||||||
@@ -60,6 +61,8 @@ def _get_feed(channel_id: str) -> bytes:
|
|||||||
)
|
)
|
||||||
if resp.status_code not in _RETRYABLE_STATUS:
|
if resp.status_code not in _RETRYABLE_STATUS:
|
||||||
break
|
break
|
||||||
|
if error is None: # cannot happen: the range above is never empty, and every try records one
|
||||||
|
raise RssError(f"RSS fetch for {channel_id} made no attempt", "transport")
|
||||||
raise error
|
raise error
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -42,6 +42,9 @@ _HEADERS = {
|
|||||||
"Accept-Language": "en-US,en;q=0.9",
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# The top-level keys a first page (contents) and a continuation page (onResponseReceivedCommands)
|
||||||
|
# put their result list under.
|
||||||
|
_RESULT_FRAMES = {"contents", "onResponseReceivedCommands"}
|
||||||
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
|
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
|
||||||
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
|
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
|
||||||
|
|
||||||
@@ -65,29 +68,36 @@ def _runs_text(node: dict | None) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def _get_config(client: httpx.Client) -> tuple[str, str]:
|
def _get_config(client: httpx.Client) -> tuple[str, str]:
|
||||||
"""The current InnerTube (api_key, client_version), cached after the first fetch."""
|
"""The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them.
|
||||||
|
|
||||||
|
The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no
|
||||||
|
keys on it) at the wrong moment pinned every later search to a hard-coded key and version for
|
||||||
|
the life of the process — and when those went stale, search returned nothing, with no error
|
||||||
|
anywhere (E32 S2). Now the next search simply tries the live pair again."""
|
||||||
global _config
|
global _config
|
||||||
if _config is not None:
|
if _config is not None:
|
||||||
return _config
|
return _config
|
||||||
try:
|
try:
|
||||||
resp = client.get(_RESULTS_URL, params={"search_query": "music"})
|
html = client.get(_RESULTS_URL, params={"search_query": "music"}).text
|
||||||
html = resp.text
|
|
||||||
key = _KEY_RE.search(html)
|
|
||||||
ver = _VER_RE.search(html)
|
|
||||||
_config = (
|
|
||||||
key.group(1) if key else _FALLBACK_KEY,
|
|
||||||
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
|
|
||||||
)
|
|
||||||
except httpx.HTTPError as exc:
|
except httpx.HTTPError as exc:
|
||||||
# The fallback pair is CACHED in `_config` for the life of the process, so one failed
|
|
||||||
# fetch at the wrong moment pins every later search to a hard-coded key and version —
|
|
||||||
# and when those go stale, search returns nothing at all, with no error anywhere.
|
|
||||||
log.warning(
|
log.warning(
|
||||||
"the InnerTube config could not be fetched; falling back to the pinned key",
|
"the InnerTube config could not be fetched; falling back to the pinned key",
|
||||||
extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
|
extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
|
||||||
)
|
)
|
||||||
_config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
|
return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
|
||||||
return _config
|
key = _KEY_RE.search(html)
|
||||||
|
ver = _VER_RE.search(html)
|
||||||
|
if key and ver:
|
||||||
|
_config = (key.group(1), ver.group(1))
|
||||||
|
return _config
|
||||||
|
log.warning(
|
||||||
|
"the results page carried no InnerTube config; falling back to the pinned key",
|
||||||
|
extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"),
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
key.group(1) if key else _FALLBACK_KEY,
|
||||||
|
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _video_item(vr: dict) -> dict | None:
|
def _video_item(vr: dict) -> dict | None:
|
||||||
@@ -182,6 +192,10 @@ def search(q: str, continuation: str | None = None) -> dict:
|
|||||||
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
|
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
|
||||||
raise ScrapeError("InnerTube returned a non-JSON response") from exc
|
raise ScrapeError("InnerTube returned a non-JSON response") from exc
|
||||||
|
|
||||||
|
# Even a search with no results carries its result list's frame. Without one this is not "no
|
||||||
|
# results" but a shape we do not know (YouTube moved, or a block page in JSON) — say so.
|
||||||
|
if not isinstance(data, dict) or not _RESULT_FRAMES & data.keys():
|
||||||
|
raise ScrapeError("InnerTube returned an unexpected shape")
|
||||||
items: list[dict] = []
|
items: list[dict] = []
|
||||||
_collect_videos(data, items, set())
|
_collect_videos(data, items, set())
|
||||||
return {"items": items, "next_page_token": _find_continuation(data)}
|
return {"items": items, "next_page_token": _find_continuation(data)}
|
||||||
|
|||||||
@@ -35,4 +35,9 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./:/app
|
- ./:/app
|
||||||
working_dir: /app
|
working_dir: /app
|
||||||
command: python -m pytest
|
# The suite, then the coverage total (E32 D4: reported, no threshold). pytest's exit code is the
|
||||||
|
# lane's — the report is printed either way and never decides it.
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
|
- 'python -m pytest --cov=app --cov-report= ; rc=$$?; echo ">> backend coverage (app/) $$(python -m coverage report --format=total)% of lines"; exit $$rc'
|
||||||
|
|||||||
@@ -25,7 +25,7 @@
|
|||||||
{
|
{
|
||||||
"include": ["app", "alembic"],
|
"include": ["app", "alembic"],
|
||||||
"extraPaths": ["."],
|
"extraPaths": ["."],
|
||||||
"typeCheckingMode": "basic",
|
"typeCheckingMode": "standard",
|
||||||
"pythonVersion": "3.12",
|
"pythonVersion": "3.12",
|
||||||
"pythonPlatform": "Linux"
|
"pythonPlatform": "Linux"
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-2
@@ -2,7 +2,9 @@
|
|||||||
# Run from backend/ (the image's /app). `pythonpath = .` puts the app package on sys.path so tests
|
# Run from backend/ (the image's /app). `pythonpath = .` puts the app package on sys.path so tests
|
||||||
# import it as `app.*`, matching runtime. Most of the suite is pure-logic; the DB-backed lane opts in
|
# import it as `app.*`, matching runtime. Most of the suite is pure-logic; the DB-backed lane opts in
|
||||||
# via the `db` fixture and runs in the compose test image. `-rs` reports skip REASONS so a silently
|
# via the `db` fixture and runs in the compose test image. `-rs` reports skip REASONS so a silently
|
||||||
# skipped lane is visible (the compose lane sets SIFTLODE_TEST_REQUIRE_DB=1 to make skips a hard error).
|
# skipped lane is visible (and conftest makes a missing DB/ffmpeg a hard error unless
|
||||||
|
# SIFTLODE_TEST_REQUIRE_DB=0). `--strict-markers` (E32): a misspelt `@pytest.mark.skipiff` is an error,
|
||||||
|
# not an unknown marker that quietly does nothing.
|
||||||
testpaths = tests
|
testpaths = tests
|
||||||
pythonpath = .
|
pythonpath = .
|
||||||
addopts = -q -rs
|
addopts = -q -rs --strict-markers
|
||||||
|
|||||||
@@ -7,7 +7,10 @@
|
|||||||
# basedpyright: the backend type-check lane (E27 S3). Pinned like everything else here — a type
|
# basedpyright: the backend type-check lane (E27 S3). Pinned like everything else here — a type
|
||||||
# checker whose version floats silently changes what "clean" means, and the ratchet baseline it
|
# checker whose version floats silently changes what "clean" means, and the ratchet baseline it
|
||||||
# writes is keyed to the diagnostics THIS version produces.
|
# writes is keyed to the diagnostics THIS version produces.
|
||||||
|
# pytest-cov: the coverage REPORT the gate prints (E32 D4) — a number to read, deliberately with no
|
||||||
|
# threshold; "every regression test seen red" is the bar, not a percentage.
|
||||||
basedpyright==1.40.1
|
basedpyright==1.40.1
|
||||||
pytest==8.4.2
|
pytest==8.4.2
|
||||||
|
pytest-cov==7.1.0
|
||||||
ruff==0.15.21
|
ruff==0.15.21
|
||||||
httpx2==2.13.0
|
httpx2==2.13.0
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
# Pinned dependency lock for reproducible builds (SEC-5). GENERATED — do not hand-edit.
|
# Pinned dependency lock for reproducible builds (SEC-5). GENERATED — do not hand-edit.
|
||||||
# Source of truth is requirements.txt (the floor ranges); regenerate after editing it with:
|
# Source of truth is requirements.txt (the floor ranges); regenerate after editing it with:
|
||||||
# docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort
|
# docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort
|
||||||
# The prod Dockerfile + Dockerfile.test + CI install THIS file; pip-audit scans it in CI.
|
# The prod Dockerfile + Dockerfile.test + CI install THIS file; pip-audit scans it in CI (blocking
|
||||||
|
# since E32). A security fix may be ONE pin bumped in place instead of a full regeneration, which
|
||||||
|
# would move every other package too — then `pip install -r requirements.lock && pip check` must
|
||||||
|
# pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177, E32).
|
||||||
APScheduler==3.11.3
|
APScheduler==3.11.3
|
||||||
Authlib==1.7.2
|
Authlib==1.7.2
|
||||||
Mako==1.4.1
|
Mako==1.4.1
|
||||||
@@ -53,7 +56,7 @@ starlette==1.6.0
|
|||||||
typing-inspection==0.4.4
|
typing-inspection==0.4.4
|
||||||
typing_extensions==4.16.0
|
typing_extensions==4.16.0
|
||||||
tzlocal==5.4.4
|
tzlocal==5.4.4
|
||||||
urllib3==2.7.0
|
urllib3==2.8.0
|
||||||
uvicorn==0.52.3
|
uvicorn==0.52.3
|
||||||
uvloop==0.22.1
|
uvloop==0.22.1
|
||||||
watchfiles==1.2.0
|
watchfiles==1.2.0
|
||||||
|
|||||||
+27
-8
@@ -1,7 +1,8 @@
|
|||||||
# Ruff config for the Siftlode backend. Keeps ruff's default rule set (E4/E7/E9 + F = the
|
# Ruff config for the Siftlode backend. Keeps ruff's default rule set (E4/E7/E9 + F = the
|
||||||
# high-signal pyflakes/pycodestyle checks that catch real dead code + bugs), but ignores E402:
|
# high-signal pyflakes/pycodestyle checks that catch real dead code + bugs), all of it on. E402 was
|
||||||
# a handful of modules (e.g. main.py) deliberately run setup — logging config, sys.path — before
|
# ignored for main.py's logging setup; the ignore had let 11 incidental late imports in elsewhere
|
||||||
# importing route modules, so "module import not at top of file" is intentional there, not cruft.
|
# (E32 review), so it is on now and main.py marks each deliberate one (`# noqa: E402 — <why>`).
|
||||||
|
# `scripts/e2e_seed.py` edits `sys.path` before its imports, which E402 itself allows.
|
||||||
#
|
#
|
||||||
# line-length 100 (ruff's default is 88): matches the frontend prettier printWidth so both halves of
|
# line-length 100 (ruff's default is 88): matches the frontend prettier printWidth so both halves of
|
||||||
# the repo wrap at the same width, and it's the width `ruff format` enforces (gated via
|
# the repo wrap at the same width, and it's the width `ruff format` enforces (gated via
|
||||||
@@ -17,9 +18,10 @@ target-version = "py312"
|
|||||||
# LOG — stdlib logging misuse (`.warn()`, a logger built by hand, `exc_info` on `.exception`)
|
# LOG — stdlib logging misuse (`.warn()`, a logger built by hand, `exc_info` on `.exception`)
|
||||||
# G — the call's shape: %-style args, never an f-string the level filter cannot skip, and
|
# G — the call's shape: %-style args, never an f-string the level filter cannot skip, and
|
||||||
# never `str(exc)` where `exc_info` would carry the traceback
|
# never `str(exc)` where `exc_info` would carry the traceback
|
||||||
# T20 — `print` instead of a log line. Two deliberate ones remain, each with its `# noqa`
|
# T20 — `print` instead of a log line. Three deliberate ones remain, each with its `# noqa`
|
||||||
# and a why: `downloads/runner.py`'s usage line is for whoever typed it, and
|
# and a why: `downloads/runner.py`'s usage line is for whoever typed it, `logsink.py`
|
||||||
# `logsink.py` reports the SINK's own failure, which cannot be logged into the sink.
|
# reports the SINK's own failure, which cannot be logged into the sink, and
|
||||||
|
# `scripts/e2e_seed.py` prints the seeded ids for the e2e run that reads them.
|
||||||
# TRY400 — `.error()` in an `except` where `.exception()` would keep the traceback
|
# TRY400 — `.error()` in an `except` where `.exception()` would keep the traceback
|
||||||
# TRY401 — the exception spelled into the message AND passed as `exc_info` (it is in both)
|
# TRY401 — the exception spelled into the message AND passed as `exc_info` (it is in both)
|
||||||
# S112 — `except: continue`, the silent swallow inside a loop
|
# S112 — `except: continue`, the silent swallow inside a loop
|
||||||
@@ -29,5 +31,22 @@ target-version = "py312"
|
|||||||
# nobody to raise to, a best-effort backfill inside a sync) and always worth a sentence.
|
# nobody to raise to, a best-effort backfill inside a sync) and always worth a sentence.
|
||||||
# The wider rule these two enforce lives in tests/test_swallowed_exceptions.py: a handler that
|
# The wider rule these two enforce lives in tests/test_swallowed_exceptions.py: a handler that
|
||||||
# neither logs nor re-raises says `# silent-ok: <why>`. Ruff sees only the two narrow shapes.
|
# neither logs nor re-raises says `# silent-ok: <why>`. Ruff sees only the two narrow shapes.
|
||||||
extend-select = ["LOG", "G", "T20", "TRY400", "TRY401", "S110", "S112", "BLE001"]
|
#
|
||||||
ignore = ["E402"]
|
# E32 (gate integrity) added three more, each measured before it went on and with every hit fixed:
|
||||||
|
# RUF100 — a `# noqa` that suppresses nothing. A stale one is a claim with no rule behind it, and
|
||||||
|
# it stays armed for the day the line changes (24 were removed). A comment that merely
|
||||||
|
# explains a noqa below it names the rule as `# TRY400: …`, never `# noqa TRY400`.
|
||||||
|
# PLW1510 — `subprocess.run` without an explicit `check=`. Found the real E32 bug: three ffprobe
|
||||||
|
# probes read the output of a probe that had FAILED and took it for the file's truth.
|
||||||
|
# B — flake8-bugbear: loop-variable capture, `raise` without `from` inside `except`, a
|
||||||
|
# `zip` that silently truncates (each one is `strict=True` now — the lengths are an
|
||||||
|
# invariant, so a mismatch should be loud). B008 (a call in a default argument) stays ON:
|
||||||
|
# FastAPI's `Depends(...)` / `Query(...)` ARE that shape by design (474 hits, all of
|
||||||
|
# them), so those two are declared immutable below instead of switching the rule off —
|
||||||
|
# any other call in a default still fails.
|
||||||
|
extend-select = [
|
||||||
|
"LOG", "G", "T20", "TRY400", "TRY401", "S110", "S112", "BLE001", "RUF100", "PLW1510", "B",
|
||||||
|
]
|
||||||
|
|
||||||
|
[lint.flake8-bugbear]
|
||||||
|
extend-immutable-calls = ["fastapi.Depends", "fastapi.Query"]
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ from fastapi.testclient import TestClient
|
|||||||
from sqlalchemy import create_engine, event, text
|
from sqlalchemy import create_engine, event, text
|
||||||
from starlette.requests import HTTPConnection
|
from starlette.requests import HTTPConnection
|
||||||
|
|
||||||
import app.models # noqa: F401 — register every model on Base.metadata
|
import app.models # register every model on Base.metadata
|
||||||
import app.state as app_state
|
import app.state as app_state
|
||||||
from app import security
|
from app import security
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
@@ -37,9 +37,11 @@ from app.models import User
|
|||||||
# Sentinel for "this dependency had no override before the fixture set one" (see override_dep).
|
# Sentinel for "this dependency had no override before the fixture set one" (see override_dep).
|
||||||
_MISSING = object()
|
_MISSING = object()
|
||||||
|
|
||||||
# The compose test lane sets this so a would-be silent skip (no DB, or no ffmpeg) becomes a hard error
|
# A would-be silent skip (no DB, or no ffmpeg) is a hard error unless the run says otherwise — the
|
||||||
# — the gate must not print green while the DB-backed / ffmpeg tests were quietly not run.
|
# suite must not print green while the DB-backed / ffmpeg tests were quietly not run. ON by default
|
||||||
_REQUIRE_FULL = os.getenv("SIFTLODE_TEST_REQUIRE_DB") == "1"
|
# since E32: as an opt-in only the compose lane and CI set, a hand-run `pytest` skipped ~60 DB tests
|
||||||
|
# and still passed. `SIFTLODE_TEST_REQUIRE_DB=0` is the explicit "pure-logic only" run.
|
||||||
|
_REQUIRE_FULL = os.getenv("SIFTLODE_TEST_REQUIRE_DB", "1") != "0"
|
||||||
|
|
||||||
|
|
||||||
def _db_reachable() -> bool:
|
def _db_reachable() -> bool:
|
||||||
@@ -86,19 +88,20 @@ def _truncate_all():
|
|||||||
|
|
||||||
|
|
||||||
def pytest_configure(config):
|
def pytest_configure(config):
|
||||||
"""Fail the whole run — loudly, once — when SIFTLODE_TEST_REQUIRE_DB=1 but the environment can't
|
"""Fail the whole run — loudly, once — when the full suite is required (the default) but the
|
||||||
actually run the full suite. Without this, an unreachable DATABASE_URL or a missing ffmpeg turns
|
environment can't actually run it. Without this, an unreachable DATABASE_URL or a missing ffmpeg turns
|
||||||
~60 DB tests + the poster tests into silent skips and the gate still exits green."""
|
~60 DB tests + the poster tests into silent skips and the gate still exits green."""
|
||||||
if not _REQUIRE_FULL:
|
if not _REQUIRE_FULL:
|
||||||
return
|
return
|
||||||
if not _DB_REACHABLE:
|
if not _DB_REACHABLE:
|
||||||
raise pytest.UsageError(
|
raise pytest.UsageError(
|
||||||
"SIFTLODE_TEST_REQUIRE_DB=1 but no Postgres is reachable at the test DATABASE_URL — the "
|
"no Postgres is reachable at the test DATABASE_URL — the DB-backed lane would silently "
|
||||||
"DB-backed lane would silently skip. Fix the DB wiring or unset the flag."
|
"skip. Fix the DB wiring, or run the pure-logic tests only with SIFTLODE_TEST_REQUIRE_DB=0."
|
||||||
)
|
)
|
||||||
if shutil.which("ffmpeg") is None:
|
if shutil.which("ffmpeg") is None:
|
||||||
raise pytest.UsageError(
|
raise pytest.UsageError(
|
||||||
"SIFTLODE_TEST_REQUIRE_DB=1 but ffmpeg is missing — the thumbnail/poster tests would skip."
|
"ffmpeg is missing — the thumbnail/poster tests would skip. Install it, or run the "
|
||||||
|
"pure-logic tests only with SIFTLODE_TEST_REQUIRE_DB=0."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -36,9 +36,14 @@ def test_a_public_url_gets_past_the_guard(db, as_user, make_user, monkeypatch):
|
|||||||
"getaddrinfo",
|
"getaddrinfo",
|
||||||
lambda *a, **kw: [(None, None, None, "", ("93.184.216.34", 0))],
|
lambda *a, **kw: [(None, None, None, "", ("93.184.216.34", 0))],
|
||||||
)
|
)
|
||||||
r = as_user(user).post("/api/downloads", json={"source": "https://example.com/v"})
|
# An inline spec, so nothing ELSE refuses it: without one the request stopped at the
|
||||||
detail = r.json().get("detail") if r.status_code >= 400 else None
|
# profile lookup (400 no_profile) and "not url_blocked" held for a request the guard never
|
||||||
assert not (isinstance(detail, dict) and detail.get("code") == "url_blocked")
|
# had to wave through (E32 S2).
|
||||||
|
r = as_user(user).post(
|
||||||
|
"/api/downloads", json={"source": "https://example.com/v", "spec": {"format": "best"}}
|
||||||
|
)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
assert r.json()["status"] == "queued"
|
||||||
|
|
||||||
|
|
||||||
def test_youtube_sources_never_consult_the_guard(db, as_user, make_user, monkeypatch):
|
def test_youtube_sources_never_consult_the_guard(db, as_user, make_user, monkeypatch):
|
||||||
@@ -50,7 +55,13 @@ def test_youtube_sources_never_consult_the_guard(db, as_user, make_user, monkeyp
|
|||||||
user = make_user()
|
user = make_user()
|
||||||
db.commit()
|
db.commit()
|
||||||
monkeypatch.setattr(urlguard.socket, "getaddrinfo", boom)
|
monkeypatch.setattr(urlguard.socket, "getaddrinfo", boom)
|
||||||
as_user(user).post("/api/downloads", json={"source": "dQw4w9WgXcQ"})
|
# The job must actually be created: a request refused for any other reason would never reach
|
||||||
|
# the guard either, and the test would pass without proving anything (E32 S2).
|
||||||
|
r = as_user(user).post(
|
||||||
|
"/api/downloads", json={"source": "dQw4w9WgXcQ", "spec": {"format": "best"}}
|
||||||
|
)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
assert r.json()["source_kind"] == "youtube"
|
||||||
|
|
||||||
|
|
||||||
def test_manual_subscription_sync_goes_on_cooldown(db, as_user, make_user, monkeypatch):
|
def test_manual_subscription_sync_goes_on_cooldown(db, as_user, make_user, monkeypatch):
|
||||||
@@ -81,8 +92,12 @@ def test_the_cooldown_is_per_user_not_global(db, as_user, make_user, monkeypatch
|
|||||||
monkeypatch.setattr(syncroutes, "import_subscriptions", lambda d, u: {"imported": 0})
|
monkeypatch.setattr(syncroutes, "import_subscriptions", lambda d, u: {"imported": 0})
|
||||||
|
|
||||||
ca = as_user(a)
|
ca = as_user(a)
|
||||||
for _ in range(4):
|
assert [ca.post("/api/sync/subscriptions").status_code for _ in range(4)] == [
|
||||||
ca.post("/api/sync/subscriptions")
|
200,
|
||||||
|
200,
|
||||||
|
200,
|
||||||
|
429,
|
||||||
|
]
|
||||||
assert ca.post("/api/sync/subscriptions").status_code == 429
|
assert ca.post("/api/sync/subscriptions").status_code == 429
|
||||||
assert as_user(b).post("/api/sync/subscriptions").status_code == 200
|
assert as_user(b).post("/api/sync/subscriptions").status_code == 200
|
||||||
|
|
||||||
@@ -100,7 +115,7 @@ def test_the_single_playlist_repull_shares_the_pull_cooldown(db, as_user, make_u
|
|||||||
db.commit()
|
db.commit()
|
||||||
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
monkeypatch.setattr(plroutes, "has_read_scope", lambda u: True)
|
||||||
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: {"ok": True})
|
monkeypatch.setattr(plroutes, "repull_playlist", lambda d, u, p: {"ok": True})
|
||||||
monkeypatch.setattr(plroutes, "sync_user_playlists", lambda d, u: {"ok": True})
|
monkeypatch.setattr(plroutes, "pull_user_playlists", lambda d, u: {"ok": True})
|
||||||
|
|
||||||
c = as_user(user)
|
c = as_user(user)
|
||||||
codes = [c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code for _ in range(4)]
|
codes = [c.post(f"/api/playlists/{pl.id}/revert-youtube").status_code for _ in range(4)]
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
"""E30: an admin action that counts something keeps the count as data in its audit row's `after`, under
|
||||||
|
the key the audit log names in the reader's language (`scheduler.result.<key>`, frontend
|
||||||
|
auditColumns.tsx) — not only in the English `summary`. A renamed key would fall back to the raw word.
|
||||||
|
DB lane (Postgres).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.models import AuditLog
|
||||||
|
|
||||||
|
|
||||||
|
def _after(db, action: str) -> dict:
|
||||||
|
db.expire_all()
|
||||||
|
return db.execute(select(AuditLog.after).where(AuditLog.action == action)).scalar_one()
|
||||||
|
|
||||||
|
|
||||||
|
def test_clearing_the_audit_log_keeps_the_count_as_data(as_user, db, make_user):
|
||||||
|
admin = make_user(email="boss@example.com", role="admin")
|
||||||
|
db.add_all([AuditLog(action="sync_pause"), AuditLog(action="sync_resume")])
|
||||||
|
db.commit()
|
||||||
|
assert as_user(admin).delete("/api/admin/audit").json()["cleared"] == 2
|
||||||
|
assert _after(db, "audit_clear") == {"cleared": 2}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_demo_reset_keeps_the_count_as_data(as_user, db, make_user):
|
||||||
|
admin = make_user(email="boss@example.com", role="admin")
|
||||||
|
seeded = as_user(admin).post("/api/admin/demo/reset").json()["playlists_seeded"]
|
||||||
|
assert _after(db, "demo_reset") == {"playlists_seeded": seeded}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_discovery_purge_keeps_its_counts_as_data(as_user, db, make_user):
|
||||||
|
admin = make_user(email="boss@example.com", role="admin")
|
||||||
|
body = as_user(admin).post("/api/admin/purge-discovery").json()
|
||||||
|
body.pop("ok")
|
||||||
|
assert _after(db, "discovery_purge") == body
|
||||||
|
assert {"videos_deleted", "channels_deleted", "search_videos_deleted"} <= set(body)
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""sync/autotag's orphan cleanup (E32 S2: the module was 32% covered).
|
||||||
|
|
||||||
|
A system tag no channel carries any more is deleted — unless a saved feed view filters on it. The
|
||||||
|
views hold tag ids; a deleted tag returns under a NEW id when a channel earns it again, and the feed
|
||||||
|
drops an unknown id, so the view silently showed the whole feed under its old name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.models import Channel, ChannelTag, SavedView, Tag
|
||||||
|
from app.sync import autotag
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_orphan_system_tag_goes_but_one_a_saved_view_filters_on_stays(db, make_user):
|
||||||
|
user = make_user()
|
||||||
|
used = Tag(user_id=None, name="Gaming", category="topic")
|
||||||
|
orphan = Tag(user_id=None, name="Cooking", category="topic")
|
||||||
|
in_view = Tag(user_id=None, name="Music", category="topic")
|
||||||
|
own = Tag(user_id=user.id, name="Mine", category="other")
|
||||||
|
ch = Channel(id="UC_tags", title="T")
|
||||||
|
db.add_all([used, orphan, in_view, own, ch])
|
||||||
|
db.flush()
|
||||||
|
db.add(ChannelTag(channel_id=ch.id, tag_id=used.id, user_id=None, source="auto"))
|
||||||
|
db.add(
|
||||||
|
SavedView(
|
||||||
|
user_id=user.id, source="feed", name="Music", filters={"tags": [in_view.id], "q": ""}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# A Plex view's filters are another vocabulary; they must not break the scan.
|
||||||
|
db.add(SavedView(user_id=user.id, source="plex", name="P", filters={"genres": ["Drama"]}))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
assert autotag._cleanup_orphan_system_tags(db) == 1
|
||||||
|
db.expire_all()
|
||||||
|
names = set(db.execute(select(Tag.name)).scalars())
|
||||||
|
assert names == {"Gaming", "Music", "Mine"} # only the unreferenced system orphan went
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_malformed_saved_view_cannot_break_the_cleanup(db, make_user):
|
||||||
|
"""A view stored around the route's shape check (or before it) must not raise for everyone's job
|
||||||
|
— `"tags": 5` raised TypeError — and `True` is not tag id 1 (E32 review)."""
|
||||||
|
user = make_user()
|
||||||
|
orphan = Tag(user_id=None, name="Cooking", category="topic")
|
||||||
|
db.add(orphan)
|
||||||
|
db.flush()
|
||||||
|
for i, filters in enumerate(
|
||||||
|
({"tags": 5}, {"tags": [True]}, ["not", "a", "dict"], {"tags": None})
|
||||||
|
):
|
||||||
|
db.add(SavedView(user_id=user.id, source="feed", name=f"bad{i}", filters=filters))
|
||||||
|
db.commit()
|
||||||
|
assert autotag._cleanup_orphan_system_tags(db) == 1
|
||||||
|
db.expire_all()
|
||||||
|
assert db.execute(select(Tag).where(Tag.name == "Cooking")).first() is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_view_kept_tag_list_larger_than_a_bind_parameter_limit_still_cleans_up(db, make_user):
|
||||||
|
user = make_user()
|
||||||
|
orphan = Tag(user_id=None, name="Cooking", category="topic")
|
||||||
|
db.add(orphan)
|
||||||
|
db.flush()
|
||||||
|
huge = list(range(10_000_000, 10_070_000)) # past psycopg's 65535 bind parameters
|
||||||
|
db.add(SavedView(user_id=user.id, source="feed", name="huge", filters={"tags": huge}))
|
||||||
|
db.commit()
|
||||||
|
assert autotag._cleanup_orphan_system_tags(db) == 1
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""A YouTube topic URL's slug → the auto-tag it gives (pure). The slugs are the ones YouTube actually
|
||||||
|
links (measured on the owner's channels, 2026-09-29): a key spelled differently from YouTube's page
|
||||||
|
never matches, silently — "Humor" sat in the map for months while YouTube links …/wiki/Humour."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.sync.autotag import map_topic_slug
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("slug", "tag"),
|
||||||
|
[
|
||||||
|
("Humour", "Comedy"),
|
||||||
|
("Video_game_culture", "Gaming"),
|
||||||
|
("Electronic_music", "Music"),
|
||||||
|
("Association_football", "Sports"),
|
||||||
|
("Knowledge", "Education"),
|
||||||
|
("Physical_fitness", "Health & Fitness"),
|
||||||
|
("Television_program", "TV"),
|
||||||
|
("Lifestyle_(sociology)", None), # YouTube's catch-all: deliberately no tag
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_a_youtube_topic_slug_gives_its_tag(slug, tag):
|
||||||
|
assert map_topic_slug(slug) == tag
|
||||||
@@ -321,7 +321,7 @@ def _body_reading_functions() -> dict[tuple[str, str], dict]:
|
|||||||
continue
|
continue
|
||||||
names = [name for _, name in calls]
|
names = [name for _, name in calls]
|
||||||
limits = []
|
limits = []
|
||||||
for node, name in zip([n for n, _ in calls], names):
|
for node, name in zip([n for n, _ in calls], names, strict=True):
|
||||||
if name.startswith("request."):
|
if name.startswith("request."):
|
||||||
limits.append("<no ceiling declared>")
|
limits.append("<no ceiling declared>")
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -0,0 +1,143 @@
|
|||||||
|
"""A channel's metadata is fetched again once it is stale, not only the first time it is seen.
|
||||||
|
|
||||||
|
Banners, avatars, descriptions and counts change on YouTube, and an old banner URL stops resolving:
|
||||||
|
a channel that kept the metadata of the day it was first imported showed a dead banner forever. The
|
||||||
|
subscription import and the channel page both refresh what is older than `DETAILS_MAX_AGE`, and
|
||||||
|
leave fresh metadata alone (each refresh spends quota). DB lane.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.models import Channel, Subscription
|
||||||
|
from app.routes import channels
|
||||||
|
from app.sync import subscriptions
|
||||||
|
from app.sync.subscriptions import DETAILS_MAX_AGE, import_subscriptions
|
||||||
|
|
||||||
|
NOW = datetime.now(timezone.utc)
|
||||||
|
FRESH = NOW - timedelta(days=1)
|
||||||
|
STALE = NOW - DETAILS_MAX_AGE - timedelta(hours=1)
|
||||||
|
|
||||||
|
|
||||||
|
def _item(cid: str) -> dict:
|
||||||
|
return {
|
||||||
|
"id": cid,
|
||||||
|
"snippet": {"title": f"{cid} renamed", "publishedAt": "2010-01-01T00:00:00Z"},
|
||||||
|
"brandingSettings": {"image": {"bannerExternalUrl": f"https://yt3.example/{cid}-new"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _youtube(asked: list[list[str]], subscribed: list[str], gone: tuple[str, ...] = ()):
|
||||||
|
class _Client:
|
||||||
|
def __init__(self, db, user):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def iter_subscriptions(self):
|
||||||
|
for cid in subscribed:
|
||||||
|
yield {"channel_id": cid, "title": cid, "yt_subscription_id": f"sub-{cid}"}
|
||||||
|
|
||||||
|
def get_channels(self, ids):
|
||||||
|
asked.append(sorted(ids))
|
||||||
|
# A terminated / deleted channel is simply absent from `channels.list`'s answer.
|
||||||
|
return [_item(cid) for cid in ids if cid not in gone]
|
||||||
|
|
||||||
|
return _Client
|
||||||
|
|
||||||
|
|
||||||
|
def _channel(db, cid: str, synced: datetime | None) -> None:
|
||||||
|
db.add(
|
||||||
|
Channel(
|
||||||
|
id=cid,
|
||||||
|
title=cid,
|
||||||
|
banner_url=f"https://yt3.example/{cid}-old",
|
||||||
|
published_at=datetime(2010, 1, 1, tzinfo=timezone.utc),
|
||||||
|
details_synced_at=synced,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_import_refreshes_stale_and_missing_details_and_leaves_fresh_ones(
|
||||||
|
db, make_user, monkeypatch
|
||||||
|
):
|
||||||
|
user = make_user()
|
||||||
|
_channel(db, "UCfresh", FRESH)
|
||||||
|
_channel(db, "UCstale", STALE)
|
||||||
|
_channel(db, "UCnever", None)
|
||||||
|
db.flush()
|
||||||
|
for cid in ("UCfresh", "UCstale", "UCnever"):
|
||||||
|
db.add(Subscription(user_id=user.id, channel_id=cid))
|
||||||
|
db.commit()
|
||||||
|
asked: list[list[str]] = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
subscriptions, "YouTubeClient", _youtube(asked, ["UCfresh", "UCstale", "UCnever"])
|
||||||
|
)
|
||||||
|
|
||||||
|
result = import_subscriptions(db, user)
|
||||||
|
|
||||||
|
assert asked == [["UCnever", "UCstale"]]
|
||||||
|
assert result["channels_detailed"] == 2
|
||||||
|
stale = db.get(Channel, "UCstale")
|
||||||
|
assert stale is not None
|
||||||
|
assert stale.banner_url == "https://yt3.example/UCstale-new"
|
||||||
|
assert stale.details_synced_at is not None and stale.details_synced_at > STALE
|
||||||
|
fresh = db.get(Channel, "UCfresh")
|
||||||
|
assert fresh is not None and fresh.banner_url == "https://yt3.example/UCfresh-old"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("synced", "refreshed"), [(STALE, True), (FRESH, False)])
|
||||||
|
def test_the_channel_page_refreshes_a_stale_channel_only(
|
||||||
|
db, make_user, monkeypatch, synced, refreshed
|
||||||
|
):
|
||||||
|
user = make_user()
|
||||||
|
_channel(db, "UCpage", synced)
|
||||||
|
db.commit()
|
||||||
|
asked: list[list[str]] = []
|
||||||
|
monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, []))
|
||||||
|
|
||||||
|
body = channels.channel_detail("UCpage", user=user, db=db)
|
||||||
|
|
||||||
|
assert asked == ([["UCpage"]] if refreshed else [])
|
||||||
|
suffix = "new" if refreshed else "old"
|
||||||
|
assert body["banner_url"] == f"https://yt3.example/UCpage-{suffix}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channel_youtube_no_longer_returns_is_not_asked_for_on_every_view(
|
||||||
|
db, make_user, monkeypatch
|
||||||
|
):
|
||||||
|
"""A terminated channel is missing from `channels.list`'s answer, so nothing was stamped and the
|
||||||
|
channel page asked for it again (1 quota unit) on every single view. The miss is stamped like a
|
||||||
|
fetch, so the next ask waits for `DETAILS_MAX_AGE` like any other channel."""
|
||||||
|
user = make_user()
|
||||||
|
db.add(Channel(id="UCgone", title="UCgone"))
|
||||||
|
db.commit()
|
||||||
|
asked: list[list[str]] = []
|
||||||
|
monkeypatch.setattr(channels, "YouTubeClient", _youtube(asked, [], gone=("UCgone",)))
|
||||||
|
|
||||||
|
channels.channel_detail("UCgone", user=user, db=db)
|
||||||
|
channels.channel_detail("UCgone", user=user, db=db)
|
||||||
|
|
||||||
|
assert asked == [["UCgone"]]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_import_stamps_a_channel_youtube_no_longer_returns(db, make_user, monkeypatch):
|
||||||
|
user = make_user()
|
||||||
|
_channel(db, "UCgone", STALE)
|
||||||
|
db.flush()
|
||||||
|
db.add(Subscription(user_id=user.id, channel_id="UCgone"))
|
||||||
|
db.commit()
|
||||||
|
asked: list[list[str]] = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
subscriptions, "YouTubeClient", _youtube(asked, ["UCgone"], gone=("UCgone",))
|
||||||
|
)
|
||||||
|
|
||||||
|
import_subscriptions(db, user)
|
||||||
|
import_subscriptions(db, user)
|
||||||
|
|
||||||
|
assert asked == [["UCgone"]]
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
"""The channel manager's sums and the feed sidebar's counts read the index alone (E31 S4d).
|
||||||
|
|
||||||
|
`GET /api/channels` counts each subscribed channel's videos and sums their durations; `/feed/summary`
|
||||||
|
and `/feed/years` count the reader's videos per channel and per year under the default filters. They
|
||||||
|
used to read every video's heap page for that (681 MB on prod), a 2–6 s first open once something else
|
||||||
|
had pushed the table out of memory. Migration 0095 makes `ix_videos_channel_pub_cover` carry every
|
||||||
|
column those queries read, so Postgres answers them with Index Only Scans. This test holds the queries
|
||||||
|
and the index together: a column added to one and not the other fails it. DB lane.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
|
from app.feedquery import filtered_query
|
||||||
|
from app.models import Video
|
||||||
|
from app.routes.channels import channel_aggregates
|
||||||
|
|
||||||
|
INDEX = "ix_videos_channel_pub_cover"
|
||||||
|
|
||||||
|
# The feed's own defaults (routes/feed.py _feed_params): what the sidebar asks on a plain open.
|
||||||
|
FEED_DEFAULTS = {
|
||||||
|
"tags": [],
|
||||||
|
"tag_modes": {"language": "or", "topic": "or", "other": "or"},
|
||||||
|
"untagged": False,
|
||||||
|
"channel_ids": [],
|
||||||
|
"q": None,
|
||||||
|
"min_duration": None,
|
||||||
|
"max_duration": None,
|
||||||
|
"max_age_days": None,
|
||||||
|
"published_after": None,
|
||||||
|
"published_before": None,
|
||||||
|
"show_normal": True,
|
||||||
|
"include_shorts": False,
|
||||||
|
"include_live": False,
|
||||||
|
"show": "unwatched",
|
||||||
|
"scope": "my",
|
||||||
|
"library_source": "organic",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _index_columns(db) -> list[str]:
|
||||||
|
# Key columns first, then the INCLUDE ones, in the index's own order.
|
||||||
|
return list(
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
"SELECT a.attname FROM pg_index i "
|
||||||
|
"JOIN pg_class c ON c.oid = i.indexrelid "
|
||||||
|
"CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, n) "
|
||||||
|
"JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum "
|
||||||
|
"WHERE c.relname = :name ORDER BY k.n"
|
||||||
|
),
|
||||||
|
{"name": INDEX},
|
||||||
|
).scalars()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _video_columns(stmt) -> set[str]:
|
||||||
|
# The columns of `videos` the statement as SENT reads: the compiled SQL, not the expression tree,
|
||||||
|
# which still carries the projection a with_only_columns replaced.
|
||||||
|
sql = str(stmt.compile(dialect=postgresql.dialect()))
|
||||||
|
return set(re.findall(r"\bvideos\.(\w+)", sql))
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_covered(db, read: set[str]) -> None:
|
||||||
|
held = _index_columns(db)
|
||||||
|
assert held, f"{INDEX} is missing"
|
||||||
|
# It leads with channel_id, so it also serves every per-channel lookup the old index did.
|
||||||
|
assert held[0] == "channel_id"
|
||||||
|
assert read <= set(held), f"read but not in {INDEX}: {sorted(read - set(held))}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_channel_sums_read_only_columns_the_index_holds(db):
|
||||||
|
_assert_covered(db, _video_columns(channel_aggregates(["UC_x"])))
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_feed_sidebar_counts_read_only_columns_the_index_holds(db, make_user):
|
||||||
|
# feed_summary groups the default filter by channel, feed_years by publish date.
|
||||||
|
base, _rank = filtered_query(db, make_user(), **FEED_DEFAULTS)
|
||||||
|
_assert_covered(
|
||||||
|
db, _video_columns(base.with_only_columns(Video.channel_id, Video.published_at))
|
||||||
|
)
|
||||||
@@ -0,0 +1,302 @@
|
|||||||
|
"""E30 S3: every A→Z list the server orders is in the reader's alphabet. The database image is Alpine,
|
||||||
|
whose `en_US.utf8` is BYTE order: "Életrajzi" and "Ötödik" sorted after "Zorró" in the feed, the channel
|
||||||
|
list, the Plex grid, the genre list. Hungarian puts "Ö" after "O" and "Cs" after "Cu". Needs the DB."""
|
||||||
|
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
import app.state as state
|
||||||
|
from app.auth import current_user
|
||||||
|
from app.collation import sorted_words
|
||||||
|
from app.db import get_db
|
||||||
|
from app.main import app
|
||||||
|
from app.models import (
|
||||||
|
Channel,
|
||||||
|
Playlist,
|
||||||
|
PlaylistItem,
|
||||||
|
PlexCollection,
|
||||||
|
PlexItem,
|
||||||
|
PlexLibrary,
|
||||||
|
Subscription,
|
||||||
|
User,
|
||||||
|
Video,
|
||||||
|
)
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
|
WORDS = ["Zorró", "Életrajzi", "Ötödik", "oroszlán", "Csillag", "Cukor"]
|
||||||
|
HU = ["Cukor", "Csillag", "Életrajzi", "oroszlán", "Ötödik", "Zorró"]
|
||||||
|
EN = ["Csillag", "Cukor", "Életrajzi", "oroszlán", "Ötödik", "Zorró"]
|
||||||
|
|
||||||
|
|
||||||
|
def _user(db, lang: str) -> User:
|
||||||
|
user = User(email=f"collate-{lang}@example.com", preferences={"language": lang})
|
||||||
|
db.add(user)
|
||||||
|
db.flush()
|
||||||
|
return user
|
||||||
|
|
||||||
|
|
||||||
|
def _client(db, user, monkeypatch, override_dep) -> TestClient:
|
||||||
|
monkeypatch.setattr(state, "_configured_cache", True)
|
||||||
|
override_dep(get_db, db.info["request"])
|
||||||
|
override_dep(current_user, lambda: user)
|
||||||
|
return TestClient(app)
|
||||||
|
|
||||||
|
|
||||||
|
def test_words_sort_in_the_readers_alphabet(db):
|
||||||
|
assert sorted_words(db, WORDS, "hu") == HU
|
||||||
|
assert sorted_words(db, WORDS, "en") == EN
|
||||||
|
assert sorted_words(db, WORDS, "xx") == EN # an unknown language sorts as English
|
||||||
|
assert sorted_words(db, [], "hu") == []
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_feed(db, user):
|
||||||
|
db.add(Channel(id="UCc", title="Channel"))
|
||||||
|
db.flush()
|
||||||
|
db.add(Subscription(user_id=user.id, channel_id="UCc", priority=0, hidden=False))
|
||||||
|
db.add_all(
|
||||||
|
Video(
|
||||||
|
id=f"v{i}",
|
||||||
|
channel_id="UCc",
|
||||||
|
published_at=datetime(2024, 1, i + 1, tzinfo=timezone.utc),
|
||||||
|
title=w,
|
||||||
|
is_short=False,
|
||||||
|
live_status="none",
|
||||||
|
via_search=False,
|
||||||
|
via_explore=False,
|
||||||
|
)
|
||||||
|
for i, w in enumerate(WORDS)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_feed_title_sort_pages_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||||
|
user = _user(db, "hu")
|
||||||
|
_seed_feed(db, user)
|
||||||
|
client = _client(db, user, monkeypatch, override_dep)
|
||||||
|
|
||||||
|
full = [it["title"] for it in client.get("/api/feed?sort=title&limit=50").json()["items"]]
|
||||||
|
assert full == HU
|
||||||
|
# The cursor compares with the same collated expression, so paging walks the same order.
|
||||||
|
walked: list[str] = []
|
||||||
|
cursor = None
|
||||||
|
for _ in range(len(WORDS)):
|
||||||
|
url = "/api/feed?sort=title&limit=2" + (f"&cursor={cursor}" if cursor else "")
|
||||||
|
body = client.get(url).json()
|
||||||
|
walked += [it["title"] for it in body["items"]]
|
||||||
|
cursor = body["next_cursor"]
|
||||||
|
if not body["has_more"]:
|
||||||
|
break
|
||||||
|
assert walked == HU
|
||||||
|
desc = client.get("/api/feed?sort=title_desc&limit=50").json()["items"]
|
||||||
|
assert [it["title"] for it in desc] == HU[::-1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_channel_list_is_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||||
|
user = _user(db, "hu")
|
||||||
|
db.add_all(Channel(id=f"UC{i}", title=w) for i, w in enumerate(WORDS))
|
||||||
|
db.flush()
|
||||||
|
db.add_all(
|
||||||
|
Subscription(user_id=user.id, channel_id=f"UC{i}", priority=0, hidden=False)
|
||||||
|
for i in range(len(WORDS))
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
r = _client(db, user, monkeypatch, override_dep).get("/api/channels")
|
||||||
|
assert [c["title"] for c in r.json()] == HU
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_plex(db):
|
||||||
|
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||||
|
db.add(lib)
|
||||||
|
db.flush()
|
||||||
|
col = PlexCollection(rating_key="col1", library_id=lib.id, title="Set", child_count=len(WORDS))
|
||||||
|
db.add(col)
|
||||||
|
db.add_all(
|
||||||
|
PlexItem(
|
||||||
|
rating_key=f"m{i}",
|
||||||
|
library_id=lib.id,
|
||||||
|
kind="movie",
|
||||||
|
title=f"EN {i}",
|
||||||
|
title_hu=w,
|
||||||
|
year=2000,
|
||||||
|
genres_hu=[w],
|
||||||
|
collection_keys=["col1"],
|
||||||
|
)
|
||||||
|
for i, w in enumerate(WORDS)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _plex(db, monkeypatch, override_dep) -> TestClient:
|
||||||
|
user = _user(db, "hu")
|
||||||
|
_seed_plex(db)
|
||||||
|
return _client(db, user, monkeypatch, override_dep)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_plex_grid_sorts_by_the_hungarian_title(db, monkeypatch, override_dep):
|
||||||
|
client = _plex(db, monkeypatch, override_dep)
|
||||||
|
grid = client.get("/api/plex/library?scope=movie&sort=title&sort_dir=asc&lang=hu").json()
|
||||||
|
assert [it["title"] for it in grid["items"]] == HU
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_plex_genre_list_is_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||||
|
client = _plex(db, monkeypatch, override_dep)
|
||||||
|
facets = client.get("/api/plex/facets?scope=movie&lang=hu").json()
|
||||||
|
assert [g["value"] for g in facets["genres"]] == HU
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_collection_sorts_by_the_hungarian_title(db, monkeypatch, override_dep):
|
||||||
|
client = _plex(db, monkeypatch, override_dep)
|
||||||
|
members = client.get("/api/plex/collections/col1?sort=title&sort_dir=asc&lang=hu").json()
|
||||||
|
assert [it["title"] for it in members["items"]] == HU
|
||||||
|
# year ties everywhere, so the default (year) order falls through to the title tiebreak
|
||||||
|
default = client.get("/api/plex/collections/col1?lang=hu").json()
|
||||||
|
assert [it["title"] for it in default["items"]] == HU
|
||||||
|
|
||||||
|
|
||||||
|
def _seed_discovery(db, user, synced: bool):
|
||||||
|
"""Channels the user's playlist holds one video from each, not subscribed: discovery lists them,
|
||||||
|
count-tied, so the title decides. `synced=False` leaves them stubs, which sends the endpoint down
|
||||||
|
its enrich path — the one that re-sorts in Python and has to agree with the ORDER BY."""
|
||||||
|
pl = Playlist(user_id=user.id, name="Mix")
|
||||||
|
db.add(pl)
|
||||||
|
db.flush()
|
||||||
|
stamp = datetime(2024, 1, 1, tzinfo=timezone.utc) if synced else None
|
||||||
|
for i, w in enumerate(WORDS):
|
||||||
|
db.add(Channel(id=f"UCd{i}", title=w, details_synced_at=stamp))
|
||||||
|
db.flush()
|
||||||
|
db.add(
|
||||||
|
Video(
|
||||||
|
id=f"d{i}",
|
||||||
|
channel_id=f"UCd{i}",
|
||||||
|
published_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||||
|
title="x",
|
||||||
|
is_short=False,
|
||||||
|
live_status="none",
|
||||||
|
via_search=False,
|
||||||
|
via_explore=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.flush()
|
||||||
|
db.add(PlaylistItem(playlist_id=pl.id, video_id=f"d{i}", position=i))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def test_channel_discovery_is_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||||
|
user = _user(db, "hu")
|
||||||
|
_seed_discovery(db, user, synced=True)
|
||||||
|
r = _client(db, user, monkeypatch, override_dep).get("/api/channels/discovery")
|
||||||
|
assert [c["title"] for c in r.json()] == HU
|
||||||
|
|
||||||
|
|
||||||
|
def test_channel_discovery_keeps_that_order_after_enriching(db, monkeypatch, override_dep):
|
||||||
|
import app.routes.channels as channels
|
||||||
|
|
||||||
|
class FakeYouTube:
|
||||||
|
def __init__(self, *a, **k): ...
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *a):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get_channels(self, ids):
|
||||||
|
return []
|
||||||
|
|
||||||
|
monkeypatch.setattr(channels, "YouTubeClient", FakeYouTube)
|
||||||
|
monkeypatch.setattr(channels, "apply_channel_details", lambda *a, **k: None)
|
||||||
|
user = _user(db, "hu")
|
||||||
|
_seed_discovery(db, user, synced=False)
|
||||||
|
r = _client(db, user, monkeypatch, override_dep).get("/api/channels/discovery")
|
||||||
|
assert [c["title"] for c in r.json()] == HU
|
||||||
|
|
||||||
|
|
||||||
|
# --- The other lists the server puts in the reader's alphabet (E30 review): each call site, once. ---
|
||||||
|
|
||||||
|
|
||||||
|
def _others(db, n_offset: int = 0) -> list[User]:
|
||||||
|
users = [
|
||||||
|
User(email=f"{w.lower()}@example.com", display_name=w) for w in WORDS
|
||||||
|
] # emails and names in WORDS' order
|
||||||
|
db.add_all(users)
|
||||||
|
db.flush()
|
||||||
|
return users
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_share_and_message_recipients_are_in_the_readers_alphabet(
|
||||||
|
db, monkeypatch, override_dep
|
||||||
|
):
|
||||||
|
from app.models import MessageKey
|
||||||
|
|
||||||
|
user = _user(db, "hu")
|
||||||
|
others = _others(db)
|
||||||
|
db.add_all(
|
||||||
|
MessageKey(
|
||||||
|
user_id=u.id,
|
||||||
|
public_key="k",
|
||||||
|
wrapped_private_key="w",
|
||||||
|
salt="s",
|
||||||
|
wrap_iv="i",
|
||||||
|
key_check="c",
|
||||||
|
)
|
||||||
|
for u in others
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
client = _client(db, user, monkeypatch, override_dep)
|
||||||
|
emails = [r["email"] for r in client.get("/api/downloads/recipients").json()]
|
||||||
|
assert emails == [f"{w.lower()}@example.com" for w in HU]
|
||||||
|
names = [r["name"] for r in client.get("/api/messages/users").json()]
|
||||||
|
assert names == HU
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_blocked_channels_are_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||||
|
from app.models import BlockedChannel
|
||||||
|
|
||||||
|
user = _user(db, "hu")
|
||||||
|
db.add_all(Channel(id=f"UC{i}", title=w) for i, w in enumerate(WORDS))
|
||||||
|
db.flush()
|
||||||
|
db.add_all(BlockedChannel(user_id=user.id, channel_id=f"UC{i}") for i in range(len(WORDS)))
|
||||||
|
db.commit()
|
||||||
|
r = _client(db, user, monkeypatch, override_dep).get("/api/channels/blocked/list")
|
||||||
|
assert [c["title"] for c in r.json()] == HU
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_plex_libraries_and_collections_are_in_the_readers_alphabet(
|
||||||
|
db, monkeypatch, override_dep
|
||||||
|
):
|
||||||
|
user = _user(db, "hu")
|
||||||
|
libs = [
|
||||||
|
PlexLibrary(plex_key=str(i), title=w, kind="movie", enabled=True)
|
||||||
|
for i, w in enumerate(WORDS)
|
||||||
|
]
|
||||||
|
db.add_all(libs)
|
||||||
|
db.flush()
|
||||||
|
db.add_all(
|
||||||
|
PlexCollection(rating_key=f"c{i}", library_id=libs[0].id, title=w, child_count=1)
|
||||||
|
for i, w in enumerate(WORDS)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
client = _client(db, user, monkeypatch, override_dep)
|
||||||
|
assert [lib["title"] for lib in client.get("/api/plex/libraries").json()["libraries"]] == HU
|
||||||
|
cols = client.get("/api/plex/collections").json()["collections"]
|
||||||
|
assert [c["title"] for c in cols] == HU
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_persons_titles_of_one_year_are_in_the_readers_alphabet(db, monkeypatch, override_dep):
|
||||||
|
user = _user(db, "hu")
|
||||||
|
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||||
|
db.add(lib)
|
||||||
|
db.flush()
|
||||||
|
db.add_all(
|
||||||
|
PlexItem(
|
||||||
|
rating_key=f"m{i}",
|
||||||
|
library_id=lib.id,
|
||||||
|
kind="movie",
|
||||||
|
title=f"EN {i}",
|
||||||
|
title_hu=w,
|
||||||
|
year=2000,
|
||||||
|
cast_names=["Anna"],
|
||||||
|
)
|
||||||
|
for i, w in enumerate(WORDS)
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
body = _client(db, user, monkeypatch, override_dep).get("/api/plex/person/Anna?lang=hu").json()
|
||||||
|
assert [m["title"] for m in body["movies"]] == HU
|
||||||
@@ -151,7 +151,7 @@ def test_nobody_reaches_another_users_session(db, make_user, as_user):
|
|||||||
def test_the_list_is_the_users_own_newest_first(db, make_user, as_user):
|
def test_the_list_is_the_users_own_newest_first(db, make_user, as_user):
|
||||||
client = as_user(make_user())
|
client = as_user(make_user())
|
||||||
first = _start(client)["id"]
|
first = _start(client)["id"]
|
||||||
client.post(f"/api/debug-sessions/{first}/stop")
|
assert client.post(f"/api/debug-sessions/{first}/stop").status_code == 200
|
||||||
second = _start(client)["id"]
|
second = _start(client)["id"]
|
||||||
assert [s["id"] for s in client.get("/api/debug-sessions").json()["items"]] == [second, first]
|
assert [s["id"] for s in client.get("/api/debug-sessions").json()["items"]] == [second, first]
|
||||||
|
|
||||||
@@ -199,10 +199,13 @@ def test_an_erased_account_takes_its_sessions_with_it(db, make_user, as_user):
|
|||||||
user = make_user()
|
user = make_user()
|
||||||
client = as_user(user)
|
client = as_user(user)
|
||||||
sid = _start(client)["id"]
|
sid = _start(client)["id"]
|
||||||
client.post(
|
assert (
|
||||||
f"/api/debug-sessions/{sid}/screenshots",
|
client.post(
|
||||||
content=_PNG,
|
f"/api/debug-sessions/{sid}/screenshots",
|
||||||
headers={"content-type": "image/png"},
|
content=_PNG,
|
||||||
|
headers={"content-type": "image/png"},
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
purge_user(db, user, BackgroundTasks())
|
purge_user(db, user, BackgroundTasks())
|
||||||
assert db.get(DebugSession, sid) is None
|
assert db.get(DebugSession, sid) is None
|
||||||
@@ -250,7 +253,7 @@ def test_a_request_naming_the_own_open_session_is_tagged_and_raised(gated, db, c
|
|||||||
_signed_in(client, make_user)
|
_signed_in(client, make_user)
|
||||||
sid = _start(client, "DEBUG")["id"]
|
sid = _start(client, "DEBUG")["id"]
|
||||||
gated.records.clear()
|
gated.records.clear()
|
||||||
client.get("/api/debug-sessions", headers={"x-debug-session": sid})
|
assert client.get("/api/debug-sessions", headers={"x-debug-session": sid}).status_code == 200
|
||||||
(line,) = _request_lines(gated)
|
(line,) = _request_lines(gated)
|
||||||
assert getattr(line, "debug_session_id", None) == sid
|
assert getattr(line, "debug_session_id", None) == sid
|
||||||
# Raised to the session's level: the line is kept for storage though the user's own is Alap.
|
# Raised to the session's level: the line is kept for storage though the user's own is Alap.
|
||||||
@@ -263,7 +266,7 @@ def test_a_header_naming_anything_else_is_ignored_not_refused(gated, db, client,
|
|||||||
db.commit()
|
db.commit()
|
||||||
_signed_in(client, make_user)
|
_signed_in(client, make_user)
|
||||||
own = _start(client)["id"]
|
own = _start(client)["id"]
|
||||||
client.post(f"/api/debug-sessions/{own}/stop")
|
assert client.post(f"/api/debug-sessions/{own}/stop").status_code == 200
|
||||||
for value in (foreign, own, "not-a-session", "f" * 16, "A" * 16):
|
for value in (foreign, own, "not-a-session", "f" * 16, "A" * 16):
|
||||||
gated.records.clear()
|
gated.records.clear()
|
||||||
# Not `/api/me`: a polled route logs at DEBUG, and an un-raised request would drop the very
|
# Not `/api/me`: a polled route logs at DEBUG, and an un-raised request would drop the very
|
||||||
@@ -280,7 +283,7 @@ def test_a_header_naming_the_own_session_past_its_cap_is_ignored(gated, db, clie
|
|||||||
sid = _start(client)["id"]
|
sid = _start(client)["id"]
|
||||||
_expire(db, sid) # past its cap, not yet swept: still open in the table
|
_expire(db, sid) # past its cap, not yet swept: still open in the table
|
||||||
gated.records.clear()
|
gated.records.clear()
|
||||||
client.get("/api/debug-sessions", headers={"x-debug-session": sid})
|
assert client.get("/api/debug-sessions", headers={"x-debug-session": sid}).status_code == 200
|
||||||
(line,) = _request_lines(gated)
|
(line,) = _request_lines(gated)
|
||||||
assert getattr(line, "debug_session_id", None) is None
|
assert getattr(line, "debug_session_id", None) is None
|
||||||
assert getattr(line, "elevated", False) is False
|
assert getattr(line, "elevated", False) is False
|
||||||
@@ -616,7 +619,7 @@ def test_a_users_screenshots_are_capped_across_their_sessions(recording, monkeyp
|
|||||||
client, sid = recording
|
client, sid = recording
|
||||||
monkeypatch.setattr(debugsession, "SCREENSHOT_USER_MAX_BYTES", len(_PNG) * 2)
|
monkeypatch.setattr(debugsession, "SCREENSHOT_USER_MAX_BYTES", len(_PNG) * 2)
|
||||||
assert _upload(client, sid, _PNG).status_code == 200
|
assert _upload(client, sid, _PNG).status_code == 200
|
||||||
client.post(f"/api/debug-sessions/{sid}/stop")
|
assert client.post(f"/api/debug-sessions/{sid}/stop").status_code == 200
|
||||||
other = _start(client)["id"]
|
other = _start(client)["id"]
|
||||||
assert _upload(client, other, _PNG).status_code == 200
|
assert _upload(client, other, _PNG).status_code == 200
|
||||||
r = _upload(client, other, _PNG)
|
r = _upload(client, other, _PNG)
|
||||||
@@ -631,7 +634,7 @@ def test_new_sessions_are_paced_but_joining_one_is_not(db, make_user, as_user, m
|
|||||||
client = as_user(make_user())
|
client = as_user(make_user())
|
||||||
sid = _start(client)["id"]
|
sid = _start(client)["id"]
|
||||||
assert _start(client)["id"] == sid # another tab joining
|
assert _start(client)["id"] == sid # another tab joining
|
||||||
client.post(f"/api/debug-sessions/{sid}/stop")
|
assert client.post(f"/api/debug-sessions/{sid}/stop").status_code == 200
|
||||||
r = client.post("/api/debug-sessions", json={"level": "DEBUG"})
|
r = client.post("/api/debug-sessions", json={"level": "DEBUG"})
|
||||||
assert r.status_code == 429 and r.json()["detail"]["reason"] == "debug_session_rate_limited"
|
assert r.status_code == 429 and r.json()["detail"]["reason"] == "debug_session_rate_limited"
|
||||||
|
|
||||||
@@ -745,9 +748,16 @@ def test_the_export_is_the_saved_draft_the_evidence_and_the_screenshots(db, make
|
|||||||
db.commit()
|
db.commit()
|
||||||
_upload(client, sid, _PNG)
|
_upload(client, sid, _PNG)
|
||||||
_upload(client, sid, _JPEG)
|
_upload(client, sid, _JPEG)
|
||||||
client.put(
|
assert (
|
||||||
f"/api/debug-sessions/{sid}/draft",
|
client.put(
|
||||||
json={"title": "Spinner stays", "body": "It spins.", "excluded": [dropped_web.id, 999999]},
|
f"/api/debug-sessions/{sid}/draft",
|
||||||
|
json={
|
||||||
|
"title": "Spinner stays",
|
||||||
|
"body": "It spins.",
|
||||||
|
"excluded": [dropped_web.id, 999999],
|
||||||
|
},
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
|
|
||||||
r = client.get(f"/api/debug-sessions/{sid}/export")
|
r = client.get(f"/api/debug-sessions/{sid}/export")
|
||||||
@@ -1144,7 +1154,7 @@ def test_new_issues_are_paced_per_user(gitea_fake, db, make_user, as_user, monke
|
|||||||
first = _start(client)["id"]
|
first = _start(client)["id"]
|
||||||
_ready(client, first)
|
_ready(client, first)
|
||||||
assert client.post(f"/api/debug-sessions/{first}/issue").status_code == 200
|
assert client.post(f"/api/debug-sessions/{first}/issue").status_code == 200
|
||||||
client.post(f"/api/debug-sessions/{first}/stop")
|
assert client.post(f"/api/debug-sessions/{first}/stop").status_code == 200
|
||||||
second = _start(client)["id"]
|
second = _start(client)["id"]
|
||||||
_ready(client, second)
|
_ready(client, second)
|
||||||
r = client.post(f"/api/debug-sessions/{second}/issue")
|
r = client.post(f"/api/debug-sessions/{second}/issue")
|
||||||
|
|||||||
@@ -212,7 +212,12 @@ def test_a_timestamp_outside_the_consent_window_is_clamped(elevated):
|
|||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
future = (now + timedelta(days=365)).isoformat()
|
future = (now + timedelta(days=365)).isoformat()
|
||||||
past = (now - timedelta(days=30)).isoformat()
|
past = (now - timedelta(days=30)).isoformat()
|
||||||
client.post("/api/diagnostics/events", json={"events": [_event(ts=future), _event(ts=past)]})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/api/diagnostics/events", json={"events": [_event(ts=future), _event(ts=past)]}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
late, early = rows()
|
late, early = rows()
|
||||||
assert late.ts <= datetime.now(timezone.utc)
|
assert late.ts <= datetime.now(timezone.utc)
|
||||||
assert early.ts >= now - timedelta(hours=diagnostics.ELEVATION_HOURS, minutes=1)
|
assert early.ts >= now - timedelta(hours=diagnostics.ELEVATION_HOURS, minutes=1)
|
||||||
@@ -342,15 +347,15 @@ def _app() -> TestClient:
|
|||||||
|
|
||||||
|
|
||||||
def test_the_account_reaches_the_handlers_own_lines(gated):
|
def test_the_account_reaches_the_handlers_own_lines(gated):
|
||||||
_app().get("/api/thing/41")
|
assert _app().get("/api/thing/41").status_code == 200
|
||||||
line = next(r for r in gated.records if r.getMessage() == "an ordinary line")
|
line = next(r for r in gated.records if r.getMessage() == "an ordinary line")
|
||||||
assert getattr(line, "user_id", None) == 41
|
assert getattr(line, "user_id", None) == 41
|
||||||
|
|
||||||
|
|
||||||
def test_one_users_elevation_logs_their_requests_in_detail_and_nobody_elses(gated):
|
def test_one_users_elevation_logs_their_requests_in_detail_and_nobody_elses(gated):
|
||||||
client = _app()
|
client = _app()
|
||||||
client.get("/api/thing/7", params={"level": "DEBUG"})
|
assert client.get("/api/thing/7", params={"level": "DEBUG"}).status_code == 200
|
||||||
client.get("/api/thing/8")
|
assert client.get("/api/thing/8").status_code == 200
|
||||||
detail = [r for r in gated.records if r.getMessage().startswith("detail only")]
|
detail = [r for r in gated.records if r.getMessage().startswith("detail only")]
|
||||||
assert [r.user_id for r in detail] == [7]
|
assert [r.user_id for r in detail] == [7]
|
||||||
assert getattr(detail[0], "elevated", False) is True
|
assert getattr(detail[0], "elevated", False) is True
|
||||||
@@ -363,9 +368,9 @@ def test_one_users_elevation_logs_their_requests_in_detail_and_nobody_elses(gate
|
|||||||
|
|
||||||
def test_the_root_level_returns_to_the_base_when_the_elevation_ends(gated):
|
def test_the_root_level_returns_to_the_base_when_the_elevation_ends(gated):
|
||||||
client = _app()
|
client = _app()
|
||||||
client.get("/api/thing/7", params={"level": "TRACE"})
|
assert client.get("/api/thing/7", params={"level": "TRACE"}).status_code == 200
|
||||||
assert logging.getLogger().level == logs.TRACE
|
assert logging.getLogger().level == logs.TRACE
|
||||||
client.get("/api/thing/7") # the same user, back at the base
|
assert client.get("/api/thing/7").status_code == 200 # the same user, back at the base
|
||||||
assert logging.getLogger().level == logging.INFO
|
assert logging.getLogger().level == logging.INFO
|
||||||
logs._track_elevation(9, logging.DEBUG, datetime.now(timezone.utc) - timedelta(seconds=1))
|
logs._track_elevation(9, logging.DEBUG, datetime.now(timezone.utc) - timedelta(seconds=1))
|
||||||
assert logging.getLogger().level == logging.INFO # an already-expired one is pruned at once
|
assert logging.getLogger().level == logging.INFO # an already-expired one is pruned at once
|
||||||
@@ -373,15 +378,25 @@ def test_the_root_level_returns_to_the_base_when_the_elevation_ends(gated):
|
|||||||
|
|
||||||
def test_the_query_string_is_logged_only_for_a_traced_request_and_redacted(gated):
|
def test_the_query_string_is_logged_only_for_a_traced_request_and_redacted(gated):
|
||||||
client = _app()
|
client = _app()
|
||||||
client.get("/api/thing/7", params={"level": "DEBUG", "q": "lofi"})
|
assert client.get("/api/thing/7", params={"level": "DEBUG", "q": "lofi"}).status_code == 200
|
||||||
client.get("/api/thing/7", params={"level": "TRACE", "q": "lofi", "token": "SECRET"})
|
assert (
|
||||||
|
client.get(
|
||||||
|
"/api/thing/7", params={"level": "TRACE", "q": "lofi", "token": "SECRET"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
lines = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
lines = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||||
assert "query" not in lines[0].attrs
|
assert "query" not in lines[0].attrs
|
||||||
assert "q=lofi" in lines[1].attrs["query"] and "SECRET" not in lines[1].attrs["query"]
|
assert "q=lofi" in lines[1].attrs["query"] and "SECRET" not in lines[1].attrs["query"]
|
||||||
|
|
||||||
|
|
||||||
def test_an_address_typed_into_a_search_is_redacted_though_the_browser_encodes_it(gated):
|
def test_an_address_typed_into_a_search_is_redacted_though_the_browser_encodes_it(gated):
|
||||||
_app().get("/api/thing/7?level=TRACE&q=someone%40example.com&new_password=hunter2")
|
assert (
|
||||||
|
_app()
|
||||||
|
.get("/api/thing/7?level=TRACE&q=someone%40example.com&new_password=hunter2")
|
||||||
|
.status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||||
assert line.attrs["query"] == "level=TRACE&q=[email]&new_password=[redacted]"
|
assert line.attrs["query"] == "level=TRACE&q=[email]&new_password=[redacted]"
|
||||||
|
|
||||||
@@ -400,7 +415,7 @@ def test_a_credential_param_goes_whole_however_its_value_decodes(raw):
|
|||||||
|
|
||||||
def test_an_admin_level_of_trace_does_not_log_other_users_query_strings(gated):
|
def test_an_admin_level_of_trace_does_not_log_other_users_query_strings(gated):
|
||||||
logs.set_level("TRACE")
|
logs.set_level("TRACE")
|
||||||
_app().get("/api/thing/7", params={"q": "something private"})
|
assert _app().get("/api/thing/7", params={"q": "something private"}).status_code == 200
|
||||||
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||||
assert "query" not in line.attrs
|
assert "query" not in line.attrs
|
||||||
|
|
||||||
@@ -488,7 +503,7 @@ def test_a_real_sign_in_carries_the_consented_level_into_the_process(gated, clie
|
|||||||
user = _signed_in(client, make_user)
|
user = _signed_in(client, make_user)
|
||||||
assert client.put("/api/me/diagnostics", json={"level": "DEBUG"}).status_code == 200
|
assert client.put("/api/me/diagnostics", json={"level": "DEBUG"}).status_code == 200
|
||||||
logs._elevations.clear() # only a request resolving the account through auth may put it back
|
logs._elevations.clear() # only a request resolving the account through auth may put it back
|
||||||
client.get("/api/me")
|
assert client.get("/api/me").status_code == 200
|
||||||
assert logs._elevations[user.id][0] == logging.DEBUG
|
assert logs._elevations[user.id][0] == logging.DEBUG
|
||||||
assert client.put("/api/me/diagnostics", json={"level": "INFO"}).status_code == 200
|
assert client.put("/api/me/diagnostics", json={"level": "INFO"}).status_code == 200
|
||||||
assert user.id not in logs._elevations
|
assert user.id not in logs._elevations
|
||||||
|
|||||||
@@ -180,7 +180,7 @@ def test_continue_watching_merges_both_sources_by_recency(db):
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
got = rails.continue_watching(db, user)
|
got = rails.continue_watching(db, user, lang="en")
|
||||||
assert [(c["source"], c["id"]) for c in got] == [("plex", "pm1"), ("youtube", "vmed")]
|
assert [(c["source"], c["id"]) for c in got] == [("plex", "pm1"), ("youtube", "vmed")]
|
||||||
assert got[0]["remaining_seconds"] == 100 * 60 - 600
|
assert got[0]["remaining_seconds"] == 100 * 60 - 600
|
||||||
|
|
||||||
@@ -208,7 +208,7 @@ def test_continue_watching_is_youtube_only_when_plex_is_off(db):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
assert {c["source"] for c in rails.continue_watching(db, user)} == {"youtube"}
|
assert {c["source"] for c in rails.continue_watching(db, user, lang="en")} == {"youtube"}
|
||||||
|
|
||||||
|
|
||||||
def test_the_resume_rail_obeys_the_channel_blocklist(db):
|
def test_the_resume_rail_obeys_the_channel_blocklist(db):
|
||||||
@@ -227,13 +227,13 @@ def test_the_resume_rail_obeys_the_channel_blocklist(db):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
assert [c["id"] for c in rails.continue_watching(db, user)] == ["vmed"]
|
assert [c["id"] for c in rails.continue_watching(db, user, lang="en")] == ["vmed"]
|
||||||
|
|
||||||
db.add(BlockedChannel(user_id=user.id, channel_id=ch.id))
|
db.add(BlockedChannel(user_id=user.id, channel_id=ch.id))
|
||||||
db.commit()
|
db.commit()
|
||||||
assert rails.continue_watching(db, user) == []
|
assert rails.continue_watching(db, user, lang="en") == []
|
||||||
# And so the hero, whose resume tier draws from that rail, has nothing to offer either.
|
# And so the hero, whose resume tier draws from that rail, has nothing to offer either.
|
||||||
assert hero.pick(db, user, "any", 0) is None
|
assert hero.pick(db, user, "any", 0, lang="en") is None
|
||||||
|
|
||||||
|
|
||||||
def test_a_null_timestamp_sorts_last(db):
|
def test_a_null_timestamp_sorts_last(db):
|
||||||
@@ -263,7 +263,7 @@ def test_a_null_timestamp_sorts_last(db):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
got = rails.continue_watching(db, user)
|
got = rails.continue_watching(db, user, lang="en")
|
||||||
assert got[0]["id"] == "vshort"
|
assert got[0]["id"] == "vshort"
|
||||||
assert len(got) == 3
|
assert len(got) == 3
|
||||||
|
|
||||||
@@ -283,7 +283,7 @@ def test_hero_prefers_resume_and_labels_the_tier(db):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
got = hero.pick(db, user, "any", seed=1)
|
got = hero.pick(db, user, "any", seed=1, lang="en")
|
||||||
assert got["tier"] == "resume"
|
assert got["tier"] == "resume"
|
||||||
assert got["card"]["id"] == "vmed"
|
assert got["card"]["id"] == "vmed"
|
||||||
|
|
||||||
@@ -302,7 +302,7 @@ def test_hero_skips_a_resume_that_does_not_fit_the_budget(db):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
got = hero.pick(db, user, "short", seed=1)
|
got = hero.pick(db, user, "short", seed=1, lang="en")
|
||||||
assert got["tier"] == "surprise"
|
assert got["tier"] == "surprise"
|
||||||
assert got["card"]["duration_seconds"] <= BANDS["short"][1]
|
assert got["card"]["duration_seconds"] <= BANDS["short"][1]
|
||||||
|
|
||||||
@@ -312,7 +312,7 @@ def test_hero_falls_back_to_the_shelf_before_guessing(db):
|
|||||||
user, ch = _seed_yt(db)
|
user, ch = _seed_yt(db)
|
||||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
||||||
db.commit()
|
db.commit()
|
||||||
got = hero.pick(db, user, "medium", seed=1)
|
got = hero.pick(db, user, "medium", seed=1, lang="en")
|
||||||
assert got["tier"] == "comfort"
|
assert got["tier"] == "comfort"
|
||||||
assert got["card"]["id"] == "vmed"
|
assert got["card"]["id"] == "vmed"
|
||||||
|
|
||||||
@@ -332,7 +332,7 @@ def test_hero_resolves_a_show_to_the_episode_that_would_play(db):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
got = hero.pick(db, user, "any", seed=1)
|
got = hero.pick(db, user, "any", seed=1, lang="en")
|
||||||
assert got["tier"] == "resume"
|
assert got["tier"] == "resume"
|
||||||
assert got["card"]["kind"] == "episode"
|
assert got["card"]["kind"] == "episode"
|
||||||
assert got["card"]["id"] == "ps1e2" # ep1 is finished → ep2 is what plays
|
assert got["card"]["id"] == "ps1e2" # ep1 is finished → ep2 is what plays
|
||||||
@@ -345,17 +345,19 @@ def test_hero_is_none_when_nothing_fits(db):
|
|||||||
user, ch = _seed_yt(db)
|
user, ch = _seed_yt(db)
|
||||||
db.execute(text("DELETE FROM videos WHERE id = 'vlong'"))
|
db.execute(text("DELETE FROM videos WHERE id = 'vlong'"))
|
||||||
db.commit()
|
db.commit()
|
||||||
assert hero.pick(db, user, "long", seed=1) is None
|
assert hero.pick(db, user, "long", seed=1, lang="en") is None
|
||||||
|
|
||||||
|
|
||||||
def test_surprise_is_reproducible_for_a_seed(db):
|
def test_surprise_is_reproducible_for_a_seed(db):
|
||||||
"""The reroll button depends on this: one seed drives both sources, so the same seed must give
|
"""The reroll button depends on this: one seed drives both sources, so the same seed must give
|
||||||
the same answer and a different seed must be free to give another."""
|
the same answer and a different seed must be free to give another."""
|
||||||
user, _ = _seed_yt(db)
|
user, _ = _seed_yt(db)
|
||||||
a = rails.surprise(db, user, "any", seed=7, limit=3)
|
a = rails.surprise(db, user, "any", seed=7, limit=3, lang="en")
|
||||||
b = rails.surprise(db, user, "any", seed=7, limit=3)
|
b = rails.surprise(db, user, "any", seed=7, limit=3, lang="en")
|
||||||
assert [c["id"] for c in a] == [c["id"] for c in b]
|
assert [c["id"] for c in a] == [c["id"] for c in b]
|
||||||
seeds = {tuple(c["id"] for c in rails.surprise(db, user, "any", s, 3)) for s in range(12)}
|
seeds = {
|
||||||
|
tuple(c["id"] for c in rails.surprise(db, user, "any", s, 3, lang="en")) for s in range(12)
|
||||||
|
}
|
||||||
assert len(seeds) > 1 # the seed actually reaches the ordering
|
assert len(seeds) > 1 # the seed actually reaches the ordering
|
||||||
|
|
||||||
|
|
||||||
@@ -374,7 +376,9 @@ def test_surprise_never_offers_something_already_started(db):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
assert "vlong" not in {c["id"] for c in rails.surprise(db, user, "long", seed=3, limit=5)}
|
assert "vlong" not in {
|
||||||
|
c["id"] for c in rails.surprise(db, user, "long", seed=3, limit=5, lang="en")
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_the_seed_decides_which_source_leads(db):
|
def test_the_seed_decides_which_source_leads(db):
|
||||||
@@ -384,7 +388,9 @@ def test_the_seed_decides_which_source_leads(db):
|
|||||||
user, _ = _seed_yt(db)
|
user, _ = _seed_yt(db)
|
||||||
_seed_plex(db, user)
|
_seed_plex(db, user)
|
||||||
_enable_plex(db)
|
_enable_plex(db)
|
||||||
leaders = {rails.surprise(db, user, "any", s, limit=1)[0]["source"] for s in range(6)}
|
leaders = {
|
||||||
|
rails.surprise(db, user, "any", s, limit=1, lang="en")[0]["source"] for s in range(6)
|
||||||
|
}
|
||||||
assert leaders == {"youtube", "plex"}
|
assert leaders == {"youtube", "plex"}
|
||||||
|
|
||||||
|
|
||||||
@@ -413,10 +419,10 @@ def test_the_reroll_must_be_able_to_move_past_the_resume_tier(db):
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
first = hero.pick(db, user, "any", seed=1)
|
first = hero.pick(db, user, "any", seed=1, lang="en")
|
||||||
assert first["card"]["id"] == "vmed"
|
assert first["card"]["id"] == "vmed"
|
||||||
# Same seed, same everything — only the exclusion differs.
|
# Same seed, same everything — only the exclusion differs.
|
||||||
second = hero.pick(db, user, "any", seed=1, exclude={"youtube:vmed"})
|
second = hero.pick(db, user, "any", seed=1, exclude={"youtube:vmed"}, lang="en")
|
||||||
assert second["card"]["id"] == "vshort"
|
assert second["card"]["id"] == "vshort"
|
||||||
|
|
||||||
|
|
||||||
@@ -438,7 +444,7 @@ def test_the_reroll_rotates_which_tier_answers(db):
|
|||||||
)
|
)
|
||||||
db.add(ComfortPin(user_id=user.id, video_id="vlong", position=0))
|
db.add(ComfortPin(user_id=user.id, video_id="vlong", position=0))
|
||||||
db.commit()
|
db.commit()
|
||||||
tiers = [hero.pick(db, user, "any", seed=1, start=n)["tier"] for n in range(4)]
|
tiers = [hero.pick(db, user, "any", seed=1, start=n, lang="en")["tier"] for n in range(4)]
|
||||||
assert tiers == ["resume", "comfort", "surprise", "resume"]
|
assert tiers == ["resume", "comfort", "surprise", "resume"]
|
||||||
|
|
||||||
|
|
||||||
@@ -447,7 +453,7 @@ def test_the_rotation_still_falls_through_an_empty_tier(db):
|
|||||||
Here there is no shelf at all, so the comfort-led press has to answer from somewhere else."""
|
Here there is no shelf at all, so the comfort-led press has to answer from somewhere else."""
|
||||||
user, _ = _seed_yt(db)
|
user, _ = _seed_yt(db)
|
||||||
assert db.execute(select(func.count()).select_from(ComfortPin)).scalar_one() == 0
|
assert db.execute(select(func.count()).select_from(ComfortPin)).scalar_one() == 0
|
||||||
got = hero.pick(db, user, "any", seed=1, start=1)
|
got = hero.pick(db, user, "any", seed=1, start=1, lang="en")
|
||||||
assert got is not None and got["tier"] != "comfort"
|
assert got is not None and got["tier"] != "comfort"
|
||||||
|
|
||||||
|
|
||||||
@@ -468,10 +474,10 @@ def test_excluding_a_show_also_excludes_the_episode_it_resolves_to(db):
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
shown = hero.pick(db, user, "any", seed=1)
|
shown = hero.pick(db, user, "any", seed=1, lang="en")
|
||||||
assert shown["card"]["id"] == "ps1e1" # the show resolved to its in-progress episode
|
assert shown["card"]["id"] == "ps1e1" # the show resolved to its in-progress episode
|
||||||
# Excluding what the USER saw — the show — must move the tier on, not return the episode again.
|
# Excluding what the USER saw — the show — must move the tier on, not return the episode again.
|
||||||
after = hero.pick(db, user, "any", seed=1, exclude={"plex:ps1"})
|
after = hero.pick(db, user, "any", seed=1, exclude={"plex:ps1"}, lang="en")
|
||||||
assert after is None or after["card"]["id"] != "ps1e1"
|
assert after is None or after["card"]["id"] != "ps1e1"
|
||||||
|
|
||||||
|
|
||||||
@@ -518,7 +524,7 @@ def test_shelf_resolves_a_channel_to_its_newest_unwatched_video(db):
|
|||||||
user, ch = _seed_yt(db)
|
user, ch = _seed_yt(db)
|
||||||
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=0))
|
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=0))
|
||||||
db.commit()
|
db.commit()
|
||||||
(card,) = pins.shelf(db, user)
|
(card,) = pins.shelf(db, user, lang="en")
|
||||||
assert card["kind"] == "channel"
|
assert card["kind"] == "channel"
|
||||||
assert card["unwatched_count"] == 3
|
assert card["unwatched_count"] == 3
|
||||||
assert card["next"]["id"] == "vshort" # newest
|
assert card["next"]["id"] == "vshort" # newest
|
||||||
@@ -538,7 +544,7 @@ def test_shelf_resolves_a_playlist_to_its_first_unwatched_item(db):
|
|||||||
db.add(VideoState(user_id=user.id, video_id="vshort", status="watched", watched_at=NOW))
|
db.add(VideoState(user_id=user.id, video_id="vshort", status="watched", watched_at=NOW))
|
||||||
db.add(ComfortPin(user_id=user.id, playlist_id=pl.id, position=0))
|
db.add(ComfortPin(user_id=user.id, playlist_id=pl.id, position=0))
|
||||||
db.commit()
|
db.commit()
|
||||||
(card,) = pins.shelf(db, user)
|
(card,) = pins.shelf(db, user, lang="en")
|
||||||
assert card["kind"] == "playlist"
|
assert card["kind"] == "playlist"
|
||||||
assert card["item_count"] == 2
|
assert card["item_count"] == 2
|
||||||
assert card["next"]["id"] == "vmed"
|
assert card["next"]["id"] == "vmed"
|
||||||
@@ -585,7 +591,7 @@ def test_the_shelf_resolves_each_channel_and_playlist_on_its_own_terms(db):
|
|||||||
|
|
||||||
got = [
|
got = [
|
||||||
(c.get("unwatched_count", c.get("item_count")), (c["next"] or {}).get("id"))
|
(c.get("unwatched_count", c.get("item_count")), (c["next"] or {}).get("id"))
|
||||||
for c in pins.shelf(db, user)
|
for c in pins.shelf(db, user, lang="en")
|
||||||
]
|
]
|
||||||
assert got == [(3, "vshort"), (1, "vother"), (1, "vlong"), (0, None)]
|
assert got == [(3, "vshort"), (1, "vother"), (1, "vlong"), (0, None)]
|
||||||
|
|
||||||
@@ -601,14 +607,14 @@ def test_the_shelf_costs_the_same_queries_for_one_pin_or_many(db):
|
|||||||
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=0))
|
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=0))
|
||||||
db.add(ComfortPin(user_id=user.id, playlist_id=lists[0].id, position=1))
|
db.add(ComfortPin(user_id=user.id, playlist_id=lists[0].id, position=1))
|
||||||
db.commit()
|
db.commit()
|
||||||
few = _count_statements(db, lambda: pins.shelf(db, user))
|
few = _count_statements(db, lambda: pins.shelf(db, user, lang="en"))
|
||||||
|
|
||||||
for i, c in enumerate(extra):
|
for i, c in enumerate(extra):
|
||||||
db.add(ComfortPin(user_id=user.id, channel_id=c.id, position=10 + i))
|
db.add(ComfortPin(user_id=user.id, channel_id=c.id, position=10 + i))
|
||||||
for i, p in enumerate(lists[1:]):
|
for i, p in enumerate(lists[1:]):
|
||||||
db.add(ComfortPin(user_id=user.id, playlist_id=p.id, position=20 + i))
|
db.add(ComfortPin(user_id=user.id, playlist_id=p.id, position=20 + i))
|
||||||
db.commit()
|
db.commit()
|
||||||
assert _count_statements(db, lambda: pins.shelf(db, user)) == few
|
assert _count_statements(db, lambda: pins.shelf(db, user, lang="en")) == few
|
||||||
|
|
||||||
|
|
||||||
def test_a_pinned_video_survives_leaving_the_feed_scope(db):
|
def test_a_pinned_video_survives_leaving_the_feed_scope(db):
|
||||||
@@ -619,7 +625,7 @@ def test_a_pinned_video_survives_leaving_the_feed_scope(db):
|
|||||||
db.commit()
|
db.commit()
|
||||||
db.execute(delete(Subscription))
|
db.execute(delete(Subscription))
|
||||||
db.commit()
|
db.commit()
|
||||||
(card,) = pins.shelf(db, user)
|
(card,) = pins.shelf(db, user, lang="en")
|
||||||
assert card["id"] == "vmed"
|
assert card["id"] == "vmed"
|
||||||
assert card["next"]["id"] == "vmed"
|
assert card["next"]["id"] == "vmed"
|
||||||
|
|
||||||
@@ -633,7 +639,9 @@ def test_the_plex_surprise_obeys_the_time_budget(db):
|
|||||||
_seed_plex(db, user)
|
_seed_plex(db, user)
|
||||||
_enable_plex(db)
|
_enable_plex(db)
|
||||||
drawn = lambda budget: { # noqa: E731
|
drawn = lambda budget: { # noqa: E731
|
||||||
(c["source"], c["id"]) for s in range(6) for c in rails.surprise(db, user, budget, s, 5)
|
(c["source"], c["id"])
|
||||||
|
for s in range(6)
|
||||||
|
for c in rails.surprise(db, user, budget, s, 5, lang="en")
|
||||||
}
|
}
|
||||||
assert ("plex", "pm1") not in drawn("short")
|
assert ("plex", "pm1") not in drawn("short")
|
||||||
# …and it IS offered where it fits, so this cannot pass by the rail being empty.
|
# …and it IS offered where it fits, so this cannot pass by the rail being empty.
|
||||||
@@ -658,7 +666,9 @@ def test_the_plex_surprise_never_offers_something_already_started(db):
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
drawn = {
|
drawn = {
|
||||||
(c["source"], c["id"]) for s in range(6) for c in rails.surprise(db, user, "long", s, 5)
|
(c["source"], c["id"])
|
||||||
|
for s in range(6)
|
||||||
|
for c in rails.surprise(db, user, "long", s, 5, lang="en")
|
||||||
}
|
}
|
||||||
assert ("plex", "pm1") not in drawn
|
assert ("plex", "pm1") not in drawn
|
||||||
|
|
||||||
@@ -682,7 +692,7 @@ def test_a_new_pin_lands_at_the_end_even_after_a_hole(db):
|
|||||||
|
|
||||||
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=pins.next_position(db, user)))
|
db.add(ComfortPin(user_id=user.id, channel_id=ch.id, position=pins.next_position(db, user)))
|
||||||
db.commit()
|
db.commit()
|
||||||
assert [c["kind"] for c in pins.shelf(db, user)] == ["video", "channel"]
|
assert [c["kind"] for c in pins.shelf(db, user, lang="en")] == ["video", "channel"]
|
||||||
positions = (
|
positions = (
|
||||||
db.execute(select(ComfortPin.position).order_by(ComfortPin.position)).scalars().all()
|
db.execute(select(ComfortPin.position).order_by(ComfortPin.position)).scalars().all()
|
||||||
)
|
)
|
||||||
@@ -695,7 +705,7 @@ def test_a_self_playing_pin_does_not_reference_itself(db):
|
|||||||
user, _ = _seed_yt(db)
|
user, _ = _seed_yt(db)
|
||||||
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
db.add(ComfortPin(user_id=user.id, video_id="vmed", position=0))
|
||||||
db.commit()
|
db.commit()
|
||||||
(card,) = pins.shelf(db, user)
|
(card,) = pins.shelf(db, user, lang="en")
|
||||||
assert card["next"] is not card
|
assert card["next"] is not card
|
||||||
assert "next" not in card["next"]
|
assert "next" not in card["next"]
|
||||||
|
|
||||||
@@ -718,7 +728,7 @@ def test_the_resume_rail_says_what_would_actually_play(db):
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
rail = rails.continue_watching(db, user)
|
rail = rails.continue_watching(db, user, lang="en")
|
||||||
tile = next(c for c in rail if c["kind"] == "show")
|
tile = next(c for c in rail if c["kind"] == "show")
|
||||||
assert tile["id"] == "ps1" # still the show
|
assert tile["id"] == "ps1" # still the show
|
||||||
assert tile["next"]["kind"] == "episode"
|
assert tile["next"]["kind"] == "episode"
|
||||||
@@ -762,7 +772,7 @@ def test_only_a_show_row_pays_for_a_resolution(db):
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
rail = rails.continue_watching(db, user)
|
rail = rails.continue_watching(db, user, lang="en")
|
||||||
# A video, a movie and a show — one row of each kind that can appear here, so the assertion below
|
# A video, a movie and a show — one row of each kind that can appear here, so the assertion below
|
||||||
# actually exercises both sides of the equality.
|
# actually exercises both sides of the equality.
|
||||||
assert {c["kind"] for c in rail} == {"video", "movie", "show"}
|
assert {c["kind"] for c in rail} == {"video", "movie", "show"}
|
||||||
@@ -778,7 +788,7 @@ def test_next_episode_takes_an_episode_with_no_state_row_at_all(db):
|
|||||||
_, show, ep1 = _seed_plex(db, user)
|
_, show, ep1 = _seed_plex(db, user)
|
||||||
db.add(PlexState(user_id=user.id, item_id=ep1.id, status="watched", watched_at=NOW))
|
db.add(PlexState(user_id=user.id, item_id=ep1.id, status="watched", watched_at=NOW))
|
||||||
db.commit() # ep2 has NO PlexState row
|
db.commit() # ep2 has NO PlexState row
|
||||||
assert queries.next_episode(db, user.id, show.id)["id"] == "ps1e2"
|
assert queries.next_episode(db, user.id, show.id, lang="en")["id"] == "ps1e2"
|
||||||
|
|
||||||
|
|
||||||
def test_a_finished_show_starts_again_from_the_top(db):
|
def test_a_finished_show_starts_again_from_the_top(db):
|
||||||
@@ -793,7 +803,7 @@ def test_a_finished_show_starts_again_from_the_top(db):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
assert queries.next_episode(db, user.id, show.id)["id"] == "ps1e1"
|
assert queries.next_episode(db, user.id, show.id, lang="en")["id"] == "ps1e1"
|
||||||
|
|
||||||
|
|
||||||
def test_next_episode_agrees_with_the_show_page_when_two_are_in_progress(db):
|
def test_next_episode_agrees_with_the_show_page_when_two_are_in_progress(db):
|
||||||
@@ -825,12 +835,16 @@ def test_next_episode_agrees_with_the_show_page_when_two_are_in_progress(db):
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
got = queries.next_episode(db, user.id, show.id)
|
got = queries.next_episode(db, user.id, show.id, lang="en")
|
||||||
assert got["id"] == "ps1e2"
|
assert got["id"] == "ps1e2"
|
||||||
# Asserted against the other implementation rather than against a literal, so the two cannot
|
# Asserted against the other implementation rather than against a literal, so the two cannot
|
||||||
# drift apart again without this failing.
|
# drift apart again without this failing.
|
||||||
ordered = [
|
ordered = [
|
||||||
leaf_card(ep, db.execute(select(PlexState).where(PlexState.item_id == ep.id)).scalar_one())
|
leaf_card(
|
||||||
|
ep,
|
||||||
|
db.execute(select(PlexState).where(PlexState.item_id == ep.id)).scalar_one(),
|
||||||
|
lang="en",
|
||||||
|
)
|
||||||
for ep in (ep1, ep2)
|
for ep in (ep1, ep2)
|
||||||
]
|
]
|
||||||
assert rollup.rollup(ordered)["resume"]["id"] == got["id"]
|
assert rollup.rollup(ordered)["resume"]["id"] == got["id"]
|
||||||
@@ -848,7 +862,7 @@ def test_next_episode_prefers_the_one_in_progress(db):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
assert queries.next_episode(db, user.id, show.id)["id"] == "ps1e2"
|
assert queries.next_episode(db, user.id, show.id, lang="en")["id"] == "ps1e2"
|
||||||
|
|
||||||
|
|
||||||
# ── HTTP surface ──────────────────────────────────────────────────────────────────────────────────
|
# ── HTTP surface ──────────────────────────────────────────────────────────────────────────────────
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user