Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
29c0715237 | ||
|
|
6b86f99249 | ||
|
|
3563858fd1 | ||
|
|
08e62bbb16 | ||
|
|
9454717f47 | ||
|
|
6ac9ffd45e | ||
|
|
fa55e01ef0 | ||
|
|
ab963c81d2 | ||
|
|
59e6cc027d | ||
|
|
ca94a716d8 | ||
|
|
63d01babe5 | ||
|
|
705377c797 | ||
|
|
2819049133 | ||
|
|
95b94a1d72 | ||
|
|
12077fbebc | ||
|
|
519fbe2a18 | ||
|
|
a39504e617 | ||
|
|
aac79ffa53 | ||
|
|
67c9f10153 | ||
|
|
584d082b6b | ||
|
|
f3a85916ec | ||
|
|
f6298d29af | ||
|
|
632e39d458 | ||
|
|
13d95ef70e | ||
|
|
0f9ed7cdbf | ||
|
|
ffcea0cb07 | ||
|
|
a3f8314c9a | ||
|
|
d5ccb1091a | ||
|
|
528774d640 | ||
|
|
3a9ea9e330 | ||
|
|
25f2d9d0e4 | ||
|
|
5ab6778b97 | ||
|
|
26697472cf | ||
|
|
2abd4ffcb6 | ||
|
|
a9ac303d3e | ||
|
|
740602b9bf | ||
|
|
e31a588f19 | ||
|
|
cda626be10 | ||
|
|
b7ce1d467c | ||
|
|
52fb999af6 | ||
|
|
e0453fa07d | ||
|
|
2525a12264 | ||
|
|
8e09dd85f8 | ||
|
|
addb3d1b32 | ||
|
|
86884e368e | ||
|
|
51f881681b | ||
|
|
27483ae1d6 | ||
|
|
595c29e17c | ||
|
|
acf93a1f36 | ||
|
|
84278293df | ||
|
|
dc24d80ed4 | ||
|
|
3810af66da | ||
|
|
eb339195c7 | ||
|
|
5d1065c1bc | ||
|
|
6ffb5bd8c1 | ||
|
|
dddf79ec2f | ||
|
|
5ebb4cb506 | ||
|
|
0b1d905866 | ||
|
|
52c15adaf2 | ||
|
|
3372626e34 | ||
|
|
98bb841a6a | ||
|
|
98bf0a9d66 | ||
|
|
17bb14fb18 | ||
|
|
70f98b352f | ||
|
|
efa4aeeeda | ||
|
|
a7d41eb5f3 | ||
|
|
76eead976f | ||
|
|
922d589b09 | ||
|
|
a975ab627a | ||
|
|
9bedd96774 | ||
|
|
383c85231a | ||
|
|
de62035889 | ||
|
|
b456232d00 | ||
|
|
c9e95ecea6 | ||
|
|
7987bd8a37 | ||
|
|
3411265dc1 | ||
|
|
098b8dbe43 | ||
|
|
f48f16830e | ||
|
|
955c23d321 | ||
|
|
d3e1d8219c | ||
|
|
4480839db0 | ||
|
|
8e25567ff2 | ||
|
|
8f65bd184e | ||
|
|
ae9c248987 | ||
|
|
bd6aaa1e43 | ||
|
|
2c65edfffd | ||
|
|
6c2c01e67b | ||
|
|
ef343dbec0 | ||
|
|
a58a398427 | ||
|
|
d510c8a534 | ||
|
|
dc5206f9d3 | ||
|
|
89e65ca331 | ||
|
|
17916b411a | ||
|
|
3ef2c38eb4 | ||
|
|
4c63d6660d | ||
|
|
a453b77f04 | ||
|
|
8e1004fe6f | ||
|
|
663f30feeb | ||
|
|
6653d81a80 | ||
|
|
21df7c8348 | ||
|
|
97381e65d0 | ||
|
|
f8ced6d819 | ||
|
|
f254588409 | ||
|
|
3e7c79fa69 | ||
|
|
47c12f9405 | ||
|
|
bd0914b7ed | ||
|
|
079cee4e9c | ||
|
|
edf2f22ef9 | ||
|
|
e1ef56cdbc | ||
|
|
4d0abc61d5 | ||
|
|
7c2b22bafa | ||
|
|
5e5c8a42a9 | ||
|
|
2b3094ffc5 | ||
|
|
9c91eb5d22 | ||
|
|
2df60fbf07 | ||
|
|
a2d17d693c | ||
|
|
c4917619ee | ||
|
|
760bdae278 | ||
|
|
1b946113c6 | ||
|
|
b0e51c6f89 | ||
|
|
7455fff401 | ||
|
|
aaf1843b09 | ||
|
|
a4737b92e4 | ||
|
|
75ae810866 | ||
|
|
9d7526ee57 | ||
|
|
fa0b0edd63 | ||
|
|
bfc6d199c0 | ||
|
|
4d6e59b80a | ||
|
|
4646aae320 | ||
|
|
eddeca82d3 | ||
|
|
69edeecc45 | ||
|
|
6088e476a7 | ||
|
|
ce171fa954 | ||
|
|
113da76c36 | ||
|
|
4ef11901db | ||
|
|
29abf609f3 | ||
|
|
e13d93a3ed | ||
|
|
b9850b1710 | ||
|
|
905a9f2856 | ||
|
|
1d8b83c37c | ||
|
|
62dbf35ea3 | ||
|
|
e9ccd9bc98 | ||
|
|
cc8e0648db | ||
|
|
ff80cc96fd | ||
|
|
295c1db5f1 | ||
|
|
0521149cf1 | ||
|
|
b4a6569d0d | ||
|
|
1833f594cb | ||
|
|
1615838f33 | ||
|
|
7fa8f4ebde | ||
|
|
6354454cc9 | ||
|
|
c47c193ad1 | ||
|
|
db10bfbbca | ||
|
|
dac9d89f91 | ||
|
|
81936e9f56 | ||
|
|
86f467bd65 | ||
|
|
7d480e2ad0 | ||
|
|
ad136c3fe8 | ||
|
|
9c4a999e54 | ||
|
|
3f5634a44f | ||
|
|
f4bc60d659 | ||
|
|
7055548260 | ||
|
|
477271907e | ||
|
|
50923bb684 | ||
|
|
d2e40dcffb | ||
|
|
58878958d8 | ||
|
|
b47d22cf3b | ||
|
|
e804ff3b8e | ||
|
|
cd6f119883 | ||
|
|
321776480e | ||
|
|
7c0647f87a | ||
|
|
efd7b217ca | ||
|
|
3d7ef9e945 | ||
|
|
7fce7a7ce5 | ||
|
|
f5018bdcbc | ||
|
|
b82c22b888 | ||
|
|
f839a461e6 | ||
|
|
8525666913 | ||
|
|
3f24248e01 | ||
|
|
6b86673af4 | ||
|
|
d924c665f8 | ||
|
|
aa51644f5f | ||
|
|
378b7559e4 | ||
|
|
05121fbc99 | ||
|
|
bf518223e5 | ||
|
|
332d24a646 | ||
|
|
c5e022f056 | ||
|
|
4065ac7e6a | ||
|
|
ce08fc2aec | ||
|
|
ea6d829c29 | ||
|
|
6a7ff330f6 | ||
|
|
7e1aeeb9fe | ||
|
|
7d682c90d5 | ||
|
|
cc9fc3b544 | ||
|
|
c9101e3450 | ||
|
|
51daf3e388 | ||
|
|
09d351a0e7 | ||
|
|
b5078003ce | ||
|
|
f2c32cfb8e | ||
|
|
1453fa4a33 | ||
|
|
efe106dfda | ||
|
|
ec63056bd7 | ||
|
|
09becb3d00 | ||
|
|
8d2ec5e867 | ||
|
|
0e9839e73d | ||
|
|
05ab963951 | ||
|
|
8142484347 | ||
|
|
0d91b82f86 | ||
|
|
90fc86d437 | ||
|
|
2fe58073e4 | ||
|
|
bc03e82687 | ||
|
|
3f983e6d76 | ||
|
|
134ec9a68e | ||
|
|
9cf177b198 | ||
|
|
a5fae489e0 | ||
|
|
8d0083d4e4 | ||
|
|
9f177557d2 | ||
|
|
46fce4a767 | ||
|
|
2d9595b0e5 | ||
|
|
a7d7456279 | ||
|
|
44a792c34f | ||
|
|
3987cd4260 | ||
|
|
95d9ed4aa6 | ||
|
|
8d0063d514 | ||
|
|
0ca0cac534 | ||
|
|
cf0c55ba80 | ||
|
|
fa4f823fc8 | ||
|
|
884060891e | ||
|
|
955c8c3445 | ||
|
|
99df44d7fb | ||
|
|
30fbdc52b2 | ||
|
|
046ff735cf | ||
|
|
7afaba0547 | ||
|
|
db8ed5a004 | ||
|
|
0ef25a8037 | ||
|
|
17a26894b0 | ||
|
|
946c8f26db | ||
|
|
8759b5860d | ||
|
|
dc902d2c0e | ||
|
|
cedcbb1434 | ||
|
|
4f72b193aa | ||
|
|
7b94e9a651 | ||
|
|
c6bf84a578 | ||
|
|
444f9b8166 | ||
|
|
7a47223a58 | ||
|
|
bc138e3b37 | ||
|
|
df7c3db085 | ||
|
|
6fda49369f | ||
|
|
8018cd4685 | ||
|
|
29875b7367 | ||
|
|
7ae5c57b6b | ||
|
|
dc4e8b9fcd | ||
|
|
6f9ba428d5 | ||
|
|
c60359bfed | ||
|
|
e6b76e156a | ||
|
|
e74986cc9a | ||
|
|
2df45b27d3 | ||
|
|
bacd6aec4f | ||
|
|
fad301e7a4 | ||
|
|
1cfb19bdc7 | ||
|
|
90c30d596f | ||
|
|
7ed6ccf171 | ||
|
|
ce30b99327 | ||
|
|
a7891d03c5 | ||
|
|
038da5374f | ||
|
|
92979fa920 | ||
|
|
2d5bd275fa | ||
|
|
49fe56b6e7 | ||
|
|
1fa6a0bea8 | ||
|
|
582f20fab6 | ||
|
|
6cb8231a9f | ||
|
|
6bbf532b70 | ||
|
|
2fa5c8c394 | ||
|
|
373415a2eb | ||
|
|
582c033e1c | ||
|
|
ef652ee5d8 | ||
|
|
250e3a9f90 | ||
|
|
6096dd6927 | ||
|
|
51e98bfc38 | ||
|
|
3ee00fb051 | ||
|
|
29094f5f51 | ||
|
|
16a44dec79 | ||
|
|
4e3e68d6e6 | ||
|
|
eb29f22eb7 | ||
|
|
8f6f89832f | ||
|
|
1c49870a7a | ||
|
|
142593083f | ||
|
|
8cfab498cf | ||
|
|
b5996a0e56 | ||
|
|
5e208353e3 | ||
|
|
66dbd0af81 | ||
|
|
59fd068711 | ||
|
|
8e12eee996 | ||
|
|
89c77ae6b5 | ||
|
|
0b672ed6e7 | ||
|
|
54d19cc386 | ||
|
|
04b58e5c6f | ||
|
|
6b0401b922 | ||
|
|
0cff964ff2 | ||
|
|
dc221338be | ||
|
|
8c10e57208 | ||
|
|
dd040adc96 | ||
|
|
4ab128ccab | ||
|
|
a28193a00c | ||
|
|
986bc8d01e | ||
|
|
07e3ea3737 | ||
|
|
b2eaa9e00b | ||
|
|
791ba5f507 | ||
|
|
7789db8240 | ||
|
|
f92cd9017f | ||
|
|
4bfba4aa86 | ||
|
|
86bc008a81 | ||
|
|
fac5c4465a | ||
|
|
9c9b1cf6d5 | ||
|
|
c3d88d9b20 | ||
|
|
569190480d | ||
|
|
5883ea509d | ||
|
|
a210992bfc | ||
|
|
0517de1f59 | ||
|
|
b59fd38a9c | ||
|
|
88a166e4aa | ||
|
|
07766c43e8 | ||
|
|
bb3c6dec45 | ||
|
|
3cf8f04d73 | ||
|
|
2b56c5c2eb | ||
|
|
c06a456a9a | ||
|
|
864f673ef5 | ||
|
|
6c63df6704 | ||
|
|
a5bcd9fca2 | ||
|
|
618ef812e6 | ||
|
|
e299c66121 | ||
|
|
11aef4ab7a | ||
|
|
4e52bdd4ca | ||
|
|
595490cb98 | ||
|
|
7537882235 | ||
|
|
47d968eeea | ||
|
|
e7f38396a2 | ||
|
|
f569b19c49 | ||
|
|
e795d191c5 | ||
|
|
4ad7183102 | ||
|
|
2c3aaec62d | ||
|
|
70598d47a3 | ||
|
|
5858e7b513 | ||
|
|
5346efcd10 | ||
|
|
6169e32ca5 | ||
|
|
90523284ab | ||
|
|
17bfea8754 | ||
|
|
2a9ddb78ac | ||
|
|
e887233210 | ||
|
|
b17d92d7c2 | ||
|
|
90ee34a1b4 | ||
|
|
118fc70e08 | ||
|
|
636372511b | ||
|
|
1354f5b503 | ||
|
|
98ec1ffcab | ||
|
|
18c11ca437 | ||
|
|
ca5e446fbd | ||
|
|
4c7ab12a3e | ||
|
|
c6b552e75f | ||
|
|
3cd0c525e2 | ||
|
|
f130a1c05a | ||
|
|
bbd31c7143 | ||
|
|
8731508645 | ||
|
|
b5d37ca142 | ||
|
|
eb0507f942 | ||
|
|
6d09b94bbb | ||
|
|
5274216981 | ||
|
|
75d89613d0 | ||
|
|
8fb276aa90 | ||
|
|
78a9324381 | ||
|
|
8a50538275 | ||
|
|
b4aea7fa5b | ||
|
|
92c305bab9 | ||
|
|
32fe0768b6 | ||
|
|
415a6c16f9 | ||
|
|
8c51062e5b | ||
|
|
63bf1bca78 | ||
|
|
c7a377396b | ||
|
|
8036251589 | ||
|
|
71d7646bf1 | ||
|
|
3eb0f41962 | ||
|
|
a0114deff1 | ||
|
|
d89d887a9b | ||
|
|
f8a31d48cf | ||
|
|
7f17b17c13 | ||
|
|
1f81684b6e | ||
|
|
33fddacee2 | ||
|
|
12121255dd | ||
|
|
4a7b65d187 | ||
|
|
47af4d5c05 | ||
|
|
8a83bc0e9f | ||
|
|
41b652f8fd | ||
|
|
545996f807 | ||
|
|
267fd7f70f | ||
|
|
6d49da17fe | ||
|
|
6a92585098 | ||
|
|
31dc4184f5 | ||
|
|
86971c574d | ||
|
|
785080f3da | ||
|
|
76b5f7c93e | ||
|
|
1cfe6a7f59 | ||
|
|
33fe895539 | ||
|
|
5c5cabbe5a | ||
|
|
4f37c41021 | ||
|
|
f7faa19ee2 | ||
|
|
0b4b268678 | ||
|
|
d349bc9084 | ||
|
|
dcd8a9566a | ||
|
|
6de2cd950d | ||
|
|
645d14f45d | ||
|
|
7360339039 | ||
|
|
473d9f02f2 | ||
|
|
d2e225666b | ||
|
|
efd95fca01 | ||
|
|
060deb89d2 | ||
|
|
4869d6c108 | ||
|
|
d20317e0f1 | ||
|
|
d2ba65e383 | ||
|
|
cf7ec5e9ec | ||
|
|
f76a2afbf1 | ||
|
|
889d2acf52 | ||
|
|
782362b808 | ||
|
|
5a42d76e8a | ||
|
|
ed4ab4e13c | ||
|
|
b8bff504d8 | ||
|
|
1630a3df3c |
@@ -28,6 +28,14 @@ ALLOWED_EMAILS=
|
||||
# Admin emails (subset of the above) get the admin role.
|
||||
ADMIN_EMAILS=
|
||||
|
||||
# Logging (E22). LOG_LEVEL is the BOOTSTRAP level — the level a process starts at and the only one
|
||||
# the worker/download child read; the API's running level is the admin Configuration page's
|
||||
# `log_level` knob, which takes effect without a restart. TRACE | DEBUG | INFO | WARNING | ERROR, in
|
||||
# any letter case; any other value stops the boot with an error. LOG_FORMAT is "json" (one object per
|
||||
# line — what a log collector ingests) or "text" (a readable line for a local terminal).
|
||||
LOG_LEVEL=INFO
|
||||
LOG_FORMAT=json
|
||||
|
||||
# Optional: origin of a separately-served frontend dev server (enables CORS). Leave empty in production.
|
||||
FRONTEND_ORIGIN=
|
||||
|
||||
|
||||
@@ -1,22 +1,50 @@
|
||||
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
||||
# independent of any one machine. Two parallel jobs match the two local lanes:
|
||||
# independent of any one machine. Two jobs match the two local lanes:
|
||||
# frontend — tsc (3 projects) + eslint + prettier --check + knip + vitest
|
||||
# backend — ruff + the full pytest suite (pure-logic + the DB-backed lane) against a real Postgres
|
||||
#
|
||||
# They are declared side by side but they DO NOT run in parallel: the lab has a single act_runner
|
||||
# with one slot, so they queue. Measured across runs #86-#91 — the backend job starts at the exact
|
||||
# second the frontend one ends, every time, and the pair costs ~4 + ~6 = **~10 minutes from the
|
||||
# push**. Anything that waits on this gate has to budget for the sum, not the max; the ops
|
||||
# `release` read-back timed out on its first real run for believing otherwise.
|
||||
#
|
||||
# Runner: the lab act_runner (LXC 115) mounts the Docker socket, so service containers work. The backend
|
||||
# runs pytest NATIVELY against a `postgres` service (not the local docker-compose.test.yml, whose
|
||||
# bind-mount is unreliable under a mounted-socket runner); fidelity is kept by installing the SAME pinned
|
||||
# requirements, Postgres 16, and ffmpeg, and by setting SIFTLODE_TEST_REQUIRE_DB=1 so a DB/ffmpeg that
|
||||
# failed to come up is a hard error, never a silent skip. node 22 matches the prod build image
|
||||
# (Dockerfile `node:22-alpine`).
|
||||
# failed to come up is a hard error, never a silent skip.
|
||||
#
|
||||
# The frontend container's node major is NOT free: it must equal the prod build image's and the dev
|
||||
# machine's, because vitest/tsc read the engine's own Intl and web-API implementations and those split
|
||||
# between majors. `.nvmrc` is the single source of truth and
|
||||
# `frontend/src/lib/nodeEngine.contract.test.ts` fails this gate if any of them drift apart.
|
||||
name: gate
|
||||
|
||||
on: [push, pull_request]
|
||||
# Branch pushes only. A release pushes `main` and then the `v<ver>` tag, and a bare `on: [push]`
|
||||
# treated the tag as a second push of the IDENTICAL commit — every release ran this whole gate
|
||||
# twice, to the same verdict, for nothing (runs #81-#90 are five such pairs). `branches: ['**']`
|
||||
# keeps every branch covered and drops tag refs.
|
||||
#
|
||||
# It has to be `branches`, NOT the `tags-ignore` this obviously wants to be. Gitea's matchPushEvent
|
||||
# requires EVERY named filter to match, and each one bails on the wrong ref kind: `tags-ignore`
|
||||
# starts with `if !refName.IsTag() { break }`, so a filter meant to skip tags rejects every BRANCH
|
||||
# push and the gate stops running entirely. `branches` bails the same way on a tag, which is the
|
||||
# direction we want. (Measured in go-gitea v1.27.3 modules/actions/workflows.go, not assumed — it
|
||||
# differs from GitHub, where `tags-ignore` alone leaves branches unfiltered.)
|
||||
#
|
||||
# If that filter were ever wrong the gate would not run at all — which is why `siftlode release`
|
||||
# now ends by reading the verdict back for the sha it pushed, and says so out loud when there is
|
||||
# no run for it. A gate that silently stops running is the failure this repo already lived through.
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
container: node:22-bookworm
|
||||
container: node:24-bookworm
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
@@ -34,7 +62,11 @@ jobs:
|
||||
# A combined python+node image: python 3.12 (matches the prod Dockerfile) for the tests, and node
|
||||
# so the JS actions (checkout) run. actions/setup-python is NOT usable here — the runner can't reach
|
||||
# GitHub's python-versions download, so it must be baked into the image.
|
||||
container: nikolaik/python-nodejs:python3.12-nodejs20
|
||||
#
|
||||
# This node runs the checkout action, never our code, so it is deliberately OUTSIDE the engine
|
||||
# contract the frontend job is held to — it tracks `.nvmrc` as hygiene (node 20 went EOL in April
|
||||
# 2026), but a missing upstream tag here must never be able to block the gate.
|
||||
container: nikolaik/python-nodejs:python3.12-nodejs24
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
@@ -61,6 +93,22 @@ jobs:
|
||||
run: python -m ruff check --config backend/ruff.toml backend scripts
|
||||
- name: ruff format --check
|
||||
run: python -m ruff format --check --config backend/ruff.toml backend scripts
|
||||
# The ninth lane (E27 S3). Run from backend/ so it resolves exactly as the local lane does —
|
||||
# pyrightconfig.json's include/extraPaths are relative to the config, but the baseline is
|
||||
# found relative to the working directory. New errors fail; the ~200 pre-existing ones are in
|
||||
# .basedpyright/baseline.json and burn down epic by epic. Matching is per (file, rule, column
|
||||
# span) plus a count, so a new error of the SAME shape in a file where a sibling was just
|
||||
# fixed is absorbed — burn the hot files down first.
|
||||
#
|
||||
# The second step is not redundant. basedpyright TIGHTENS the baseline in place when the count
|
||||
# drops, with no --writebaseline, and CI throws its workspace away — so without this a
|
||||
# burn-down is green here and red on the developer's `siftlode check`, which has the same
|
||||
# guard. Two gates over one tree must not answer differently.
|
||||
- name: basedpyright
|
||||
run: python -m basedpyright --project pyrightconfig.json
|
||||
working-directory: backend
|
||||
- name: basedpyright baseline is committed
|
||||
run: git diff --exit-code -- backend/.basedpyright/baseline.json
|
||||
- name: pytest
|
||||
run: python -m pytest
|
||||
working-directory: backend
|
||||
|
||||
@@ -37,3 +37,7 @@ backups/
|
||||
Thumbs.db
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
# TypeScript incremental build artifacts — machine-local, absolute-path-bearing, and no gate lane
|
||||
# reads or regenerates them, so a committed one is stale from the commit that adds it.
|
||||
*.tsbuildinfo
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
3.12
|
||||
+903
-1158
File diff suppressed because it is too large
Load Diff
@@ -7,17 +7,18 @@ worse than no rule — if you change how the code works, change this file in the
|
||||
|
||||
Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a download center). It is a
|
||||
**modular monolith**: one Postgres, one FastAPI process (uvicorn + APScheduler), one worker process
|
||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 18 + react-query frontend with a router-free navigation
|
||||
provider.
|
||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 19 (with the React Compiler) + react-query frontend, with a
|
||||
router-free navigation provider.
|
||||
|
||||
## The gate — never merge or ship red
|
||||
|
||||
The local gate is **ruff (`check` + `format --check`) + typecheck (3 tsc projects) + eslint + prettier
|
||||
The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + typecheck (3 tsc projects) + eslint + prettier
|
||||
--check + knip + vitest + pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
||||
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
||||
|
||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` + `ruff format --check backend` and `pytest` (the DB-lane needs Postgres).
|
||||
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` and burn down epic by epic. Fixed some? Regenerate with `--writebaseline`.
|
||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||
|
||||
## Workflow
|
||||
@@ -45,26 +46,29 @@ how it gets built. Concretely:
|
||||
behind each option and the weight of the decision. These decision points are where the product owner
|
||||
engages with the build and learns from it, so make them understandable — a reasoned menu, not a bare one.
|
||||
- **Mockups for design/UX decisions.** For any visual or interaction choice, show mockup options to
|
||||
choose from before building.
|
||||
choose from before building. How the UI looks and behaves is decided in
|
||||
[`docs/DESIGN.md`](docs/DESIGN.md) (its variants render in the dev-only `/design-lab`): build to it,
|
||||
and change it only through its decision log.
|
||||
- **Challenge unsound requests.** If a request is technically ill-advised and you can back it with real
|
||||
arguments, push back or propose an alternative — don't comply blindly.
|
||||
- **Verify UI in the browser before calling it done.** tsc/unit/DOM tests miss visual and interaction
|
||||
bugs; do a real visual check of any UI change.
|
||||
bugs; do a real visual check of any UI change, and run DESIGN.md's four-step Conformance check.
|
||||
- **Be terse everywhere else.** Outside those decision points, lead with the answer or decision. During
|
||||
implementation and debugging especially: no restating the same point reworded, no filler.
|
||||
|
||||
## Architecture shape (don't fight it)
|
||||
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`,
|
||||
`security`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||
its menus, `Modal`/`Overlay`, `Button`/`form`, `DataTable`/`VirtualGrid`, `QueryState`, …). That
|
||||
boundary is one-way and **lint-enforced**, not just described: a platform component may not import a
|
||||
feature component or a domain API slice (`no-restricted-imports` in `eslint.config.js`, with its own
|
||||
test). A new generic component goes in `ui/`; see [`frontend/src/components/ui/README.md`](frontend/src/components/ui/README.md).
|
||||
feature component or a domain API slice, and one layer down `lib/` may not import from `components/` at
|
||||
all, not even a type (`no-restricted-imports` in `eslint.config.js`, with its own test). A new generic
|
||||
component goes in `ui/`; see [`frontend/src/components/ui/README.md`](frontend/src/components/ui/README.md).
|
||||
Data via react-query (`qk` factory), per-account `localStorage`, and server prefs; API through
|
||||
`lib/api/*` slices over `core.req<T>`.
|
||||
|
||||
@@ -73,24 +77,37 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
|
||||
## Canonical patterns — when you have a choice, pick THIS
|
||||
|
||||
| Area | Use | Not |
|
||||
| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------ |
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`) | ad-hoc `{"unsubscribed"}`/`{"cleared": n}`/echo |
|
||||
| Transaction commit | the handler/job-wrapper commits; helpers `flush` | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE action button | `ui/Button` — pick a `SIZES` rung | a hand-rolled `<button className="px-3 py-2 …">` |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE control | the chip/segmented primitives (`Select` only for >6 options) | a native `<select>` |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| Area | Use | Not |
|
||||
| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Request SIZE | nothing — `bodylimit` caps every route (SEC-20); need more? add a ceiling to `ROUTE_MAX_BODY_BYTES` | `capped_body` on a route that does not read raw bytes itself |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE button | `ui/Button` (a `SIZES` rung) · `IconButton` · `Pressable` (a clickable card/row) · `Chip` · `DragHandle` | a raw `<button>` or `FocusSafeButton` outside the control primitives (lint-enforced) |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE text field / checkbox / slider | `ui/Input` · `Textarea` · `InnerInput` in a `fieldClass` frame · `Checkbox`/`CheckRow` · `Slider` | a raw `<input>`/`<textarea>` anywhere but those primitives (lint-enforced) |
|
||||
| FE glass surface | a role: `SURFACE.panel` / `card` / `menu` / `popover` / `tooltip` (`ui/surfaces.ts`) | a bare `glass`/`glass-card`/`glass-menu` beside a hand-picked radius (lint-enforced) |
|
||||
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
||||
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
||||
| FE effect that must not re-subscribe | `useEffectEvent` (React 19.2+), else a latest-value ref | an excused dep list asserting the same thing in prose |
|
||||
| FE hook suppression | an inline `eslint-disable-next-line` NAMING the invariant | `eslint-suppressions.json` (deliberately absent — E27 S2) |
|
||||
|
||||
## House rules that bite (learned from real bugs)
|
||||
|
||||
|
||||
+3
-1
@@ -10,7 +10,9 @@ ARG GIT_SHA=unknown
|
||||
ARG BUILD_DATE=
|
||||
|
||||
# Stage 1: build the frontend SPA
|
||||
FROM node:22-alpine AS frontend
|
||||
# The major is pinned to `.nvmrc` and held there by frontend/src/lib/nodeEngine.contract.test.ts:
|
||||
# this image, the CI gate's image and the dev machine must be ONE engine (see that test for why).
|
||||
FROM node:24-alpine AS frontend
|
||||
WORKDIR /fe
|
||||
# Deps first, keyed on the lockfile, so editing app source never re-resolves node_modules.
|
||||
COPY frontend/package.json frontend/package-lock.json ./
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,19 +0,0 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN adduser --disabled-password --gecos "" appuser && chown -R appuser /app
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["sh", "entrypoint.sh"]
|
||||
@@ -1,6 +1,8 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
# Split the path list the way the OS does (alembic's legacy space/comma/colon split is deprecated).
|
||||
path_separator = os
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from app.config import settings
|
||||
from app.db import Base
|
||||
from app.logs import configure_logging
|
||||
import app.models # noqa: F401 (import so models register on Base.metadata)
|
||||
|
||||
config = context.config
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
# Migrations run at container start (entrypoint.sh), so their output is part of the API's log
|
||||
# stream — one format with everything else, not alembic.ini's own fileConfig shape.
|
||||
configure_logging("alembic")
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""add download_jobs.error_detail — the domain-specific entry of the failure catalog (E11.5 S1)
|
||||
|
||||
Revision ID: 0084_dl_job_error_detail
|
||||
Revises: 0083_comfort_pins
|
||||
Create Date: 2026-09-13
|
||||
|
||||
`error_code` is the generic category (it decides whether a Retry is offered). `error_detail` names
|
||||
the host-specific catalog entry `downloads.errors.classify` matched on top of it — `facebook.login_wall`
|
||||
and its siblings — so the UI can say what went wrong ON THAT SITE and what to do about it, instead of
|
||||
the category's generic sentence. Nullable: most failures have no host-specific entry.
|
||||
|
||||
Backfill: a job's code is written once, at failure time, so a new rule never reaches the rows that
|
||||
failed before it. The one shape measured on prod (2026-09-13, jobs 9/66/75/145/156/157/182/256/257) is
|
||||
yt-dlp's `[facebook] <id>: Cannot parse data`, stored as `unknown`. Without a cookie that is Facebook's
|
||||
login wall. Whether a cookie was attached at the time is not recorded anywhere, so every old row gets
|
||||
the no-cookie entry — on prod no account had a Facebook cookie, so that is what they all were.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0084_dl_job_error_detail"
|
||||
down_revision: str | None = "0083_comfort_pins"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("download_jobs", sa.Column("error_detail", sa.String(length=48), nullable=True))
|
||||
op.execute(
|
||||
sa.text(
|
||||
"UPDATE download_jobs SET error_code = 'login_required', "
|
||||
"error_detail = 'facebook.login_wall' "
|
||||
"WHERE (error_code IS NULL OR error_code = 'unknown') "
|
||||
"AND error ILIKE '%[facebook]%cannot parse data%'"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("download_jobs", "error_detail")
|
||||
@@ -0,0 +1,26 @@
|
||||
"""add users.last_login_at — when the account last signed in (E22 S1)
|
||||
|
||||
Revision ID: 0085_users_last_login_at
|
||||
Revises: 0084_dl_job_error_detail
|
||||
Create Date: 2026-09-16
|
||||
|
||||
Stamped by the one sign-in path (`routes/auth._establish_session`), which the Google callback and
|
||||
password login share. No backfill is possible: the signal exists only at sign-in, so existing rows
|
||||
read NULL until each account signs in again, and the admin UI says so rather than inventing a date.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0085_users_last_login_at"
|
||||
down_revision: str | None = "0084_dl_job_error_detail"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("users", sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "last_login_at")
|
||||
@@ -0,0 +1,48 @@
|
||||
"""add log_events — stored log events for diagnostics (E22 S3)
|
||||
|
||||
Revision ID: 0086_log_events
|
||||
Revises: 0085_users_last_login_at
|
||||
Create Date: 2026-09-17
|
||||
|
||||
Written by app/logsink.py's background writer (WARNING and above, request lines), pruned by the log_gc
|
||||
job after `log_retention_days`. Sized against prod on 2026-09-16: ~2k rows on a normal day, ~14k on a
|
||||
YouTube-RSS-outage day before flood control — a few MB over the 7-day window. `user_id` carries no
|
||||
foreign key on purpose (see models.LogEvent).
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0086_log_events"
|
||||
down_revision: str | None = "0085_users_last_login_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"log_events",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True),
|
||||
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("level", sa.String(10), nullable=False),
|
||||
sa.Column("sev", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("source", sa.String(16), nullable=False),
|
||||
sa.Column("logger", sa.String(128), nullable=False),
|
||||
sa.Column("event", sa.String(64), nullable=True),
|
||||
sa.Column("msg", sa.Text(), nullable=False),
|
||||
sa.Column("attrs", postgresql.JSONB(), nullable=True),
|
||||
sa.Column("exc", sa.Text(), nullable=True),
|
||||
sa.Column("request_id", sa.String(64), nullable=True),
|
||||
sa.Column("user_id", sa.Integer(), nullable=True),
|
||||
sa.Column("debug_session_id", sa.String(64), nullable=True),
|
||||
sa.Column("job_id", sa.Integer(), nullable=True),
|
||||
sa.Column("version", sa.String(64), nullable=True),
|
||||
)
|
||||
op.create_index("ix_log_events_ts", "log_events", ["ts"])
|
||||
op.create_index("ix_log_events_request_id", "log_events", ["request_id"])
|
||||
op.create_index("ix_log_events_user_ts", "log_events", ["user_id", "ts"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("log_events")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add users.diagnostics_level + diagnostics_until — a user's consented diagnostics level (E22 S5)
|
||||
|
||||
Revision ID: 0087_users_diagnostics_level
|
||||
Revises: 0086_log_events
|
||||
Create Date: 2026-09-17
|
||||
|
||||
Both NULL = the base level (Alap). A raised level is ALWAYS paired with its expiry, computed by the
|
||||
server at consent time (`app/diagnostics.py`), so no client can ask for a level that never ends.
|
||||
Columns rather than `users.preferences`: that JSON is written wholesale by the client, and a consent
|
||||
with a lifetime must not be something a client can write.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0087_users_diagnostics_level"
|
||||
down_revision: str | None = "0086_log_events"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("users", sa.Column("diagnostics_level", sa.String(8), nullable=True))
|
||||
op.add_column(
|
||||
"users", sa.Column("diagnostics_until", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "diagnostics_until")
|
||||
op.drop_column("users", "diagnostics_level")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""add channels.feed_fallback_at — when a channel's uploads were last read through the API because
|
||||
its RSS feed would not answer
|
||||
|
||||
Revision ID: 0088_channels_feed_fallback_at
|
||||
Revises: 0087_users_diagnostics_level
|
||||
Create Date: 2026-09-19
|
||||
|
||||
YouTube's RSS feeds go dark for hours at a time, per channel (2026-09-18/19: 393 of 396 answered 404
|
||||
for four hours; the same channel answers 404 to every retry). The RSS poll then reads the newest
|
||||
uploads through the Data API instead, at a quota cost, and at most once per `rss_fallback_hours` per
|
||||
channel: this column is that clock. NULL = never needed it.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0088_channels_feed_fallback_at"
|
||||
down_revision: str | None = "0087_users_diagnostics_level"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"channels", sa.Column("feed_fallback_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("channels", "feed_fallback_at")
|
||||
@@ -0,0 +1,81 @@
|
||||
"""add debug_sessions + debug_screenshots, and index log_events.debug_session_id (E16)
|
||||
|
||||
Revision ID: 0089_debug_sessions
|
||||
Revises: 0088_channels_feed_fallback_at
|
||||
Create Date: 2026-09-19
|
||||
|
||||
A debug session is one recording: Start → reproduce → Stop (app/debugsession.py). Every log line
|
||||
of that user's requests while it runs carries its id in `log_events.debug_session_id` — the column
|
||||
E22 created and left NULL — and the issue it ends in is built from exactly those rows, so the column
|
||||
gets its index now. Partial: nearly every row has no session, and an index of NULLs would be most
|
||||
of the table for nothing.
|
||||
|
||||
At most one OPEN session per user, held by a partial unique index rather than by the start handler
|
||||
alone: two tabs pressing Start at once would otherwise both pass a "none open" check.
|
||||
|
||||
Screenshots are rows, not files: bounded (size and count per session, app/debugsession.py), gone
|
||||
with their session by cascade, and a self-host has no file volume to configure for them.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0089_debug_sessions"
|
||||
down_revision: str | None = "0088_channels_feed_fallback_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"debug_sessions",
|
||||
sa.Column("id", sa.String(16), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer,
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("level", sa.String(8), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("until", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("ended_reason", sa.String(16), nullable=True),
|
||||
sa.Column("issue_url", sa.String(512), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_debug_sessions_open_per_user",
|
||||
"debug_sessions",
|
||||
["user_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("ended_at IS NULL"),
|
||||
)
|
||||
op.create_table(
|
||||
"debug_screenshots",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column(
|
||||
"session_id",
|
||||
sa.String(16),
|
||||
sa.ForeignKey("debug_sessions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("mime", sa.String(16), nullable=False),
|
||||
sa.Column("size", sa.Integer, nullable=False),
|
||||
sa.Column("data", sa.LargeBinary, nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_log_events_debug_session",
|
||||
"log_events",
|
||||
["debug_session_id"],
|
||||
postgresql_where=sa.text("debug_session_id IS NOT NULL"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_log_events_debug_session", table_name="log_events")
|
||||
op.drop_table("debug_screenshots")
|
||||
op.drop_index("uq_debug_sessions_open_per_user", table_name="debug_sessions")
|
||||
op.drop_table("debug_sessions")
|
||||
@@ -0,0 +1,27 @@
|
||||
"""add debug_sessions.draft — the issue being written from a session (E16 S3)
|
||||
|
||||
Revision ID: 0090_debug_session_draft
|
||||
Revises: 0089_debug_sessions
|
||||
Create Date: 2026-09-19
|
||||
|
||||
The composer saves as it goes: the title, the rendered markdown body that an export or an issue
|
||||
carries, the log rows the user took out, and the form's own fields so reopening it later (Settings →
|
||||
Diagnosztika lists past sessions) shows what was written. NULL until the composer first saves.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
revision: str = "0090_debug_session_draft"
|
||||
down_revision: str | None = "0089_debug_sessions"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("debug_sessions", sa.Column("draft", JSONB, nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("debug_sessions", "draft")
|
||||
@@ -0,0 +1,27 @@
|
||||
"""add tagline_hu to plex_items + plex_shows — the HU overlay's tagline (E30 S1)
|
||||
|
||||
Revision ID: 0091_plex_tagline_hu
|
||||
Revises: 0090_debug_session_draft
|
||||
Create Date: 2026-09-29
|
||||
|
||||
The Hungarian overlay section lists a tagline beside the title and summary; the info page showed the
|
||||
English one to a Hungarian reader. NULL until the next overlay sync fills it (serving falls back to EN).
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0091_plex_tagline_hu"
|
||||
down_revision: str | None = "0090_debug_session_draft"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("plex_items", sa.Column("tagline_hu", sa.Text(), nullable=True))
|
||||
op.add_column("plex_shows", sa.Column("tagline_hu", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("plex_shows", "tagline_hu")
|
||||
op.drop_column("plex_items", "tagline_hu")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add biography_hu to plex_people — TMDB's Hungarian biography (E30 S4)
|
||||
|
||||
Revision ID: 0092_plex_person_biography_hu
|
||||
Revises: 0091_plex_tagline_hu
|
||||
Create Date: 2026-09-29
|
||||
|
||||
The person page showed a Hungarian reader the English biography although TMDB has a Hungarian one for
|
||||
most people. The person fetch now appends `translations` and keeps the Hungarian biography beside the
|
||||
English one. Already-enriched people are un-stamped, so their next page open refetches them once (the
|
||||
same lazy path as a first open) and fills it; until then the English biography stands in.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0092_plex_person_biography_hu"
|
||||
down_revision: str | None = "0091_plex_tagline_hu"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("plex_people", sa.Column("biography_hu", sa.Text(), nullable=True))
|
||||
op.execute(
|
||||
"UPDATE plex_people SET tmdb_enriched_at = NULL "
|
||||
"WHERE tmdb_person_id IS NOT NULL AND tmdb_enriched_at IS NOT NULL"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("plex_people", "biography_hu")
|
||||
@@ -0,0 +1,29 @@
|
||||
"""re-fetch linked people — "Known for" ranked by votes, without guest visits (E30 S6)
|
||||
|
||||
Revision ID: 0093_plex_person_known_for_votes
|
||||
Revises: 0092_plex_person_biography_hu
|
||||
Create Date: 2026-09-30
|
||||
|
||||
A person's "Known for" pool was their credits ranked by TMDB popularity, so talk-show visits as
|
||||
themselves crowded out the films (Tom Cruise: 14 of 15 were talk shows, none in the library). The pool is
|
||||
now ranked by vote count, without guest visits, with directed works, and deeper. Already-enriched people
|
||||
are un-stamped so the people_enrich job fetches them again; their biography stays on the row meanwhile.
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0093_plex_person_known_for_votes"
|
||||
down_revision: str | None = "0092_plex_person_biography_hu"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"UPDATE plex_people SET tmdb_enriched_at = NULL "
|
||||
"WHERE tmdb_person_id IS NOT NULL AND tmdb_enriched_at IS NOT NULL"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass # the watermark only says "fetched"; a refetch under the old ranking is not worth undoing to
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Hungarian genres on every Plex row, not only on overlaid ones (E30 review)
|
||||
|
||||
Revision ID: 0094_plex_genres_hu_everywhere
|
||||
Revises: 0093_plex_person_known_for_votes
|
||||
Create Date: 2026-09-30
|
||||
|
||||
The HU genre facets and filter read `genres_hu`, which only the overlay pass wrote — so a library with
|
||||
no HU twin, or a guid-less row, offered English genres a Hungarian filter could never match. The sync
|
||||
now writes `genres_hu` for every row (plex/sync.py `_apply_facet_fields`); this fills the rows it has
|
||||
not reached yet, from their own English list through the same map.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from app.plex.genres import hu_genres
|
||||
|
||||
revision: str = "0094_plex_genres_hu_everywhere"
|
||||
down_revision: str | None = "0093_plex_person_known_for_votes"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
for table in ("plex_items", "plex_shows"):
|
||||
rows = conn.execute(
|
||||
sa.text(
|
||||
f"SELECT id, genres FROM {table} "
|
||||
"WHERE genres_hu IS NULL AND jsonb_typeof(genres) = 'array'"
|
||||
)
|
||||
).fetchall()
|
||||
updates = [
|
||||
{"id": rid, "g": json.dumps(hu)} for rid, genres in rows if (hu := hu_genres(genres))
|
||||
]
|
||||
if updates:
|
||||
conn.execute(
|
||||
sa.text(f"UPDATE {table} SET genres_hu = CAST(:g AS jsonb) WHERE id = :id"), updates
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass # derived from `genres`; the sync writes the same values again
|
||||
@@ -13,16 +13,19 @@ Three layers, by *who* runs the code — not by feature:
|
||||
response models, status-code mapping, serialization. It owns logic that runs *only* on a request.
|
||||
Business logic that a request happens to trigger does **not** live here if anything else runs it too.
|
||||
|
||||
2. **`plex/`, `downloads/`, `sync/` — domain logic shared by more than one caller.** Anything the
|
||||
2. **`plex/`, `downloads/`, `sync/`, `issues/` — domain logic shared by more than one caller.** Anything the
|
||||
**scheduler or the worker** runs (not just a request) lives in one of these domain packages, so the
|
||||
route, the worker, and the scheduler share one definition. Examples from this codebase:
|
||||
`downloads/service.py` owns enqueue + the job state transitions (routes call it); `plex/sync.py` is
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline.
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline;
|
||||
`issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker.
|
||||
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, … are single-file primitives many
|
||||
layers import. They depend on nothing above them (a core module must never import a `routes/…`
|
||||
module).
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
||||
reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
module must never import a `routes/…` module).
|
||||
|
||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||
*second* caller needs it (a worker as well as a route; a scheduler as well as a route), extract it down
|
||||
|
||||
@@ -58,6 +58,9 @@ class AuditAction:
|
||||
DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete"
|
||||
# The log itself was cleared (a single row survives the clear as tamper-evidence)
|
||||
AUDIT_CLEAR = "audit_clear"
|
||||
# E16: a user sent a debug session's issue to the tracker — the one act that takes their log
|
||||
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
|
||||
ISSUE_CREATE = "issue_create"
|
||||
|
||||
|
||||
def record(
|
||||
|
||||
+99
-30
@@ -1,19 +1,20 @@
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from fastapi import BackgroundTasks, Depends, HTTPException, Request
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
from app import debugsession
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
from app.models import Invite, LogEvent, User
|
||||
from app.security import decrypt_optional
|
||||
from app.utils import canonical_email
|
||||
from app.refusal import refused
|
||||
|
||||
PASSWORD_MIN_LEN = 10
|
||||
|
||||
@@ -37,7 +38,7 @@ READ_SCOPES = f"{BASE_SCOPES} {READ_SCOPE}"
|
||||
WRITE_SCOPES = f"{BASE_SCOPES} {READ_SCOPE} {WRITE_SCOPE}"
|
||||
_YOUTUBE_SCOPES = frozenset({READ_SCOPE, WRITE_SCOPE})
|
||||
|
||||
log = logging.getLogger("siftlode.auth")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def google_enabled(db: Session) -> bool:
|
||||
@@ -220,7 +221,7 @@ def revoke_google_token(token: str | None) -> None:
|
||||
timeout=10,
|
||||
)
|
||||
log.info("Google token revoke: status=%s", resp.status_code)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 — a revoke we cannot make must not fail the sign-out
|
||||
log.warning("Google token revoke failed", exc_info=True)
|
||||
|
||||
|
||||
@@ -245,6 +246,12 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
||||
)
|
||||
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
|
||||
db.execute(delete(User).where(User.id == user_id))
|
||||
# Stored log events name the account by id with no foreign key to cascade from (models.LogEvent),
|
||||
# so they go explicitly — they are request history, i.e. personal data. Forgotten FIRST: this very
|
||||
# request's access line, a queued row, a later background task would otherwise put rows back
|
||||
# under the id after this DELETE (logs.forget_user).
|
||||
logs.forget_user(user_id)
|
||||
db.execute(delete(LogEvent).where(LogEvent.user_id == user_id))
|
||||
# Every canonically-equal invite goes, not just the literal spelling: is_allowed matches on the
|
||||
# canonical form, so leaving a sibling row behind would keep granting access to a mailbox whose
|
||||
# account was just erased — and prod really does carry approved pairs for one mailbox.
|
||||
@@ -269,52 +276,85 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
||||
ACTIVE_ACCOUNT_HEADER = "x-siftlode-account"
|
||||
|
||||
|
||||
def resolved_user_id(conn: HTTPConnection) -> tuple[int | None, bool]:
|
||||
def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int | None, bool]:
|
||||
"""Which account this connection acts as, and whether that's the session's default account.
|
||||
Takes any HTTPConnection — an HTTP Request OR a WebSocket — so the WS push channel shares this
|
||||
exact per-tab resolution instead of re-implementing it.
|
||||
|
||||
The signed cookie holds the *wallet* (`account_ids`, every account signed into this browser)
|
||||
plus a default `user_id`. A caller may override the default with the X-Siftlode-Account header,
|
||||
but ONLY for an account already in the wallet — so a tab can't impersonate an account that never
|
||||
authenticated here. Everything else (WebSocket, plain navigations) keeps using the cookie default.
|
||||
plus a default `user_id`. A caller may name its account with the X-Siftlode-Account header (or
|
||||
`?account=`), but ONLY an account in the wallet — so a tab can't impersonate an account that
|
||||
never authenticated here. A connection that names no account acts as the cookie default.
|
||||
|
||||
A connection naming an account NOT in the wallet acts as nobody (`None`) — typically a tab still
|
||||
pinned to an account that was signed out in another tab. It must not fall back to the default:
|
||||
that tab would go on acting as an account it never showed, its player checkpoints and pref saves
|
||||
landing there and its Sign out signing THAT account out. Only the bootstrap /api/me probe passes
|
||||
`strict=False` and gets the default, which is how such a tab learns who it now is (the frontend
|
||||
re-pins it and reloads). An account whose session ended while it stayed in the wallet (deleted,
|
||||
suspended, revoked) leaves the wallet when it is refused (`_session_user`), so a tab pinned to it
|
||||
takes the same way back.
|
||||
"""
|
||||
default_id = conn.session.get("user_id")
|
||||
wallet = conn.session.get("account_ids") or []
|
||||
# Header for normal XHR; ?account= for contexts that can't set headers (WebSocket, and a
|
||||
# plain <a> file download). Both are wallet-gated below, so neither can impersonate an
|
||||
# account that never signed into this browser.
|
||||
# plain <a> file download). Both are wallet-gated below.
|
||||
hdr = conn.headers.get(ACTIVE_ACCOUNT_HEADER) or conn.query_params.get("account")
|
||||
if hdr:
|
||||
try:
|
||||
hid = int(hdr)
|
||||
except ValueError:
|
||||
# Under `strict` this becomes a 401 and the tab authenticates as nobody, which reads
|
||||
# in the browser exactly like a session that expired. The value itself stays out —
|
||||
# it came from a request, and `wallet` says which accounts were actually on offer.
|
||||
log.debug(
|
||||
"the active-account header could not be read as an id",
|
||||
extra=logs.event("auth.account_header.invalid", accounts=len(wallet)),
|
||||
)
|
||||
hid = None
|
||||
if hid is not None and hid in wallet:
|
||||
return hid, hid == default_id
|
||||
if strict:
|
||||
return None, False
|
||||
return default_id, True
|
||||
|
||||
|
||||
def drop_from_wallet(session: dict, user_id: int) -> None:
|
||||
"""Take one account out of this browser's wallet (`account_ids`); the session default is untouched."""
|
||||
session["account_ids"] = [a for a in (session.get("account_ids") or []) if a != user_id]
|
||||
|
||||
|
||||
def _end_account_session(request: Request, user_id: int, is_default: bool) -> None:
|
||||
"""An account whose session here is no longer valid. Only nuke the whole session when it is the
|
||||
SESSION DEFAULT; a per-tab account just leaves the wallet (other tabs' default may still be valid).
|
||||
Leaving is what lets a tab pinned to it recover: its next /api/me names an account no longer signed
|
||||
in here and gets the default. Left in, it answered "signed out" — the landing page, with the default
|
||||
still signed in, and no way off it but closing the tab."""
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
else:
|
||||
drop_from_wallet(request.session, user_id)
|
||||
|
||||
|
||||
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
user_id, is_default = resolved_user_id(request)
|
||||
return _session_user(request, db, *resolved_user_id(request))
|
||||
|
||||
|
||||
def _session_user(request: Request, db: Session, user_id: int | None, is_default: bool) -> User:
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
raise refused(401, "not_authenticated")
|
||||
user = db.get(User, user_id)
|
||||
if user is None or not user.is_active or user.is_suspended:
|
||||
# Suspended/deactivated mid-session → drop the session so the block takes effect at once.
|
||||
# But only nuke the whole session when the SESSION DEFAULT identity is gone; a stale
|
||||
# per-tab header account just 401s that one tab (other tabs' default may still be valid).
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
# Suspended/deactivated mid-session → end it so the block takes effect at once.
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise refused(401, "not_authenticated")
|
||||
# SA4: reject a cookie whose recorded epoch is behind the account's current one (revoked by a
|
||||
# password reset/change or "log out everywhere"). A cookie with no recorded epoch (pre-SA4, or an
|
||||
# account signed in before this shipped) is treated as epoch 0 — so the FIRST bump revokes it too.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(user_id), 0)) != (user.session_epoch or 0):
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise refused(401, "not_authenticated")
|
||||
# Always keep the session default in the switchable list — covers sessions created before
|
||||
# multi-session existed (their account_ids was never seeded), so the switcher isn't empty.
|
||||
default_id = request.session.get("user_id")
|
||||
@@ -322,16 +362,44 @@ def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
if default_id and default_id not in accounts:
|
||||
accounts.append(default_id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
# Attribute this request's log lines to the account (E22). Here, not in the request middleware:
|
||||
# this is the one place a session becomes an account, and the middleware runs outside both the
|
||||
# session middleware and this task's context. `request.state` is shared across middleware layers
|
||||
# (it lives on the ASGI scope), so the access-log line can read it back. The user's consented
|
||||
# diagnostics level rides along: their requests are logged — and stored — in that detail (S5).
|
||||
# So does a debug session the request names (E16): only the caller's own open one counts, and a
|
||||
# header naming anything else is ignored rather than refused — a stale tab must keep working.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, until = debugsession.level_for(user, session)
|
||||
logs.set_user(user.id, level, until, session.id if session else None)
|
||||
request.state.user_id = user.id
|
||||
return user
|
||||
|
||||
|
||||
def optional_current_user(request: Request, db: Session = Depends(get_db)) -> User | None:
|
||||
"""Like `current_user` but returns None instead of raising 401 when there's no valid session.
|
||||
Used by the bootstrap /api/me probe so the logged-out landing doesn't log a failed request to
|
||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it)."""
|
||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it).
|
||||
The one lenient resolution (`strict=False`): a tab naming an account no longer signed in here gets
|
||||
the default account, which is how it learns who it now is — see `resolved_user_id`. That includes an
|
||||
account refused right here, which `_session_user` has just taken out of the wallet."""
|
||||
user_id, is_default = resolved_user_id(request, strict=False)
|
||||
try:
|
||||
return current_user(request, db)
|
||||
return _session_user(request, db, user_id, is_default)
|
||||
# silent-ok: this is the anonymous case as much as the refused one — every landing page hit
|
||||
# comes through here. The second attempt below is the handling, and it reports.
|
||||
except HTTPException:
|
||||
if is_default:
|
||||
return None
|
||||
try:
|
||||
return _session_user(request, db, *resolved_user_id(request, strict=False))
|
||||
except HTTPException:
|
||||
# Both the pinned account AND the default one were refused, so the tab is signed out
|
||||
# without being told why — and it got here from a session that named a real account.
|
||||
log.info(
|
||||
"a session was resolved to nobody: neither the named nor the default account held",
|
||||
extra=logs.event("auth.session.probe_rejected", user_id=user_id),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -341,7 +409,11 @@ def require_human(user: User = Depends(current_user)) -> User:
|
||||
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
|
||||
where there's no scope check to lean on."""
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="Not available in the demo account.")
|
||||
# Structured, because `req` leaves 403 to the caller and `notifyActionError` only shows a
|
||||
# STRUCTURED detail — a plain string here reached the demo user as their caller's generic
|
||||
# "couldn't do that", with no hint that the demo account is the reason (round 6). Affordances
|
||||
# this gate sits behind are not all UI-gated on `is_demo` (the comfort shelf is not).
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
return user
|
||||
|
||||
|
||||
@@ -351,10 +423,7 @@ def require_write_scope(user: User = Depends(require_human)) -> User:
|
||||
checks that had drifted into three different 403 messages across channels/playlists. Layers on
|
||||
`require_human` because writing to YouTube needs a real, non-demo identity."""
|
||||
if not has_write_scope(user):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Enable YouTube editing in Settings first.",
|
||||
)
|
||||
raise refused(403, "youtube_editing_off")
|
||||
return user
|
||||
|
||||
|
||||
@@ -362,7 +431,7 @@ def admin_user(user: User = Depends(current_user)) -> User:
|
||||
"""Dependency: require the signed-in user to be an admin. Single source of truth for the
|
||||
admin gate — every admin route/action depends on this rather than re-checking the role."""
|
||||
if user.role != "admin":
|
||||
raise HTTPException(status_code=403, detail="Admin only")
|
||||
raise refused(403, "admin_only")
|
||||
return user
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""The global request-body cap (SEC-20).
|
||||
|
||||
**The hole.** FastAPI reads a declared body IN FULL before auth or any other dependency runs — the
|
||||
body parameter has to be parsed to be passed in. So `POST /auth/password-login` drains a 32 MiB
|
||||
upload from an anonymous caller and only then answers 401. 70 of the 72 body-declaring routes behave
|
||||
the same way: one unauthenticated client can spend the process's memory and time at will.
|
||||
|
||||
**Why a middleware and not `capped_body` on every route.** `routes/_common.py:capped_body` is the
|
||||
right tool where a route genuinely takes bulk bytes, and the two routes that use it use it
|
||||
correctly. But putting it on all 70 others would (a) fight the canonical "Request body → a Pydantic
|
||||
model" row, since capping means taking the body as raw bytes and parsing it by hand, and (b) leave
|
||||
the NEXT route added uncovered, because nothing would remind its author. A default that has to be
|
||||
opted into is not a default. This module inverts it: the cap applies to everything, and the few
|
||||
routes that need more opt OUT, in one list, next to the code that reads it.
|
||||
|
||||
**Why raw ASGI and not `@app.middleware("http")`.** Starlette's BaseHTTPMiddleware cannot wrap
|
||||
`receive`, so it can only read `Content-Length` — which a client may omit entirely (chunked) or
|
||||
simply understate. The bytes have to be counted as they arrive, which is the same rule `capped_body`
|
||||
already applies per-route, and counting needs the receive channel.
|
||||
|
||||
**Why the refusal is byte-identical to `capped_body`'s.** Two shapes for one fact is how the
|
||||
`cancelled`/`canceled` bug happened. The 413 body here is built through the same `JSONResponse`
|
||||
FastAPI renders `HTTPException(413, {"code": "too_large"})` with — not a hand-rolled `json.dumps`,
|
||||
whose default separators differ by two bytes — and `tests/test_body_limit.py` compares the RAW BYTES
|
||||
of the two against each other rather than against a literal, so they cannot drift apart. The same
|
||||
reasoning decides where the middleware is registered (`main.py`): just inside CORS, so this 413
|
||||
carries the same headers `capped_body`'s does.
|
||||
|
||||
**No admin knob, deliberately.** The ceiling is a constant, not a `sysconfig` value: an attacker-
|
||||
facing limit that an admin can raise from the UI is a limit that can be raised by whoever gets the
|
||||
admin session, and nobody has asked to tune it. If a self-hoster ever needs a different ceiling it
|
||||
belongs in `config.py` (env, set before the process starts), not in the database.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Message, Receive, Scope, Send
|
||||
|
||||
from app import debugsession
|
||||
from app.config import settings
|
||||
|
||||
# 1 MiB. The largest body any route on the default ceiling legitimately takes is 512 KiB (the client
|
||||
# log sink, `routes/diagnostics.py:BODY_MAX_BYTES`), so this leaves 2x headroom — a ceiling that
|
||||
# refuses a legitimate request would be a worse bug than the one this module fixes. That ratio is
|
||||
# the load-bearing fact, so `test_body_limit.py` holds it rather than the figure above.
|
||||
DEFAULT_MAX_BODY_BYTES = 1024 * 1024
|
||||
|
||||
# The routes that need MORE than the default, as (method, path TEMPLATE) → their own ceiling. The
|
||||
# template, not a concrete path, so the list reads like the route decorators it mirrors and
|
||||
# `test_body_limit.py` can assert that each one still resolves to a real route: a renamed route
|
||||
# fails the gate instead of silently losing its allowance in prod.
|
||||
#
|
||||
# These are RAISED CEILINGS, NOT EXEMPTIONS, and the difference is the whole point. An exemption
|
||||
# would mean "this route is counted by something else" — which is only true when the route's own
|
||||
# limit applies to the bytes as they ARRIVE. That, in turn, is only true of a route with NO DECLARED
|
||||
# BODY PARAMETER: FastAPI parses and spools a declared body BEFORE the first dependency runs, so a
|
||||
# route that declares one has already stored the bytes by the time its own cap — or its auth — gets
|
||||
# a say.
|
||||
#
|
||||
# Both listed routes now satisfy that. The screenshot route always did (`capped_body` streams it,
|
||||
# and the owner check resolves first). The upload route did NOT until P1: it declared
|
||||
# `file: UploadFile = File(...)`, so an anonymous caller could make the process write the whole
|
||||
# per-file allowance to temp before `require_human` refused them — this middleware bounded that
|
||||
# number but could not stop the writing. It now reads its own body from a dependency declared after
|
||||
# the auth one, so its in-handler cap measures bytes as they arrive, like everything else here.
|
||||
#
|
||||
# That rule is no longer only written here: `test_body_limit.py` parses the routes and fails the
|
||||
# gate if any route in this dict declares a body parameter. A comment could not have stopped the
|
||||
# defect it describes; the test can.
|
||||
#
|
||||
# A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why
|
||||
# `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the
|
||||
# default, so listing it could only widen what it accepts.
|
||||
ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
||||
# Ingesting a local file. The body IS the file since P1 — no multipart envelope to allow for —
|
||||
# so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s
|
||||
# `download_upload_max_bytes` is bounded by this same constant (see sysconfig.SPECS), so an
|
||||
# admin can only ever LOWER the route's cap below the edge, never raise it past one.
|
||||
#
|
||||
# WHICH LAYER ANSWERS, measured rather than assumed (deep-review round 1):
|
||||
# * knob LOWERED (an override below the default) — the route's cap is the smaller number, so
|
||||
# `capped_stream_to` trips first and answers 422 `upload_too_large` carrying the admin's own
|
||||
# figure, after draining the rest of the body so the client can read it.
|
||||
# * knob at the DEFAULT — the two numbers are EQUAL, and the edge always wins: a declared
|
||||
# Content-Length over the cap is refused here before a byte is read, and an undeclared body
|
||||
# trips this counter on the same chunk the route would have tripped on. The route's 422 is
|
||||
# unreachable in that configuration, and no amount of headroom would change it — the
|
||||
# declared-length check fires first for any body big enough to matter, and the route's drain
|
||||
# pushes the count past the edge for the rest.
|
||||
#
|
||||
# So this 413 is a refusal a real user meets on a default install, not a theoretical edge, and
|
||||
# it has to be as readable as the route's own: the client localizes `too_large` (`errors.tooLarge`)
|
||||
# exactly as it does `upload_too_large`, and the upload dialog refuses an over-cap file before
|
||||
# sending it at all, off the `upload_max_bytes` that `GET /api/downloads/usage` reports.
|
||||
("POST", "/api/downloads/upload"): settings.download_upload_max_bytes,
|
||||
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
|
||||
}
|
||||
|
||||
# The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders
|
||||
# as — built through the same JSONResponse so the separators match too, and asserted against a real
|
||||
# route's raw bytes in `test_body_limit.py` rather than against a literal here.
|
||||
TOO_LARGE_BODY = JSONResponse({"detail": {"code": "too_large"}}, status_code=413).body
|
||||
|
||||
_PATH_PARAM = re.compile(r"\{[^/{}]+\}")
|
||||
|
||||
|
||||
def template_to_regex(template: str) -> re.Pattern[str]:
|
||||
"""`/api/debug-sessions/{sid}/screenshots` → a regex matching one concrete path.
|
||||
|
||||
A path parameter matches `[^/]+` — one segment, never across a `/`, so `{sid}` cannot swallow a
|
||||
deeper path and hand an exemption to a route that was never granted one.
|
||||
"""
|
||||
literals = [re.escape(part) for part in _PATH_PARAM.split(template)]
|
||||
return re.compile("^" + "[^/]+".join(literals) + "$")
|
||||
|
||||
|
||||
def _declared_length(scope: Scope) -> int | None:
|
||||
"""The `Content-Length` header as an int, or None when absent or unparseable (a chunked body has
|
||||
none, and a malformed one is not a number we may trust — both fall through to counting)."""
|
||||
for name, value in scope.get("headers", ()):
|
||||
if name == b"content-length":
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
# silent-ok: a malformed Content-Length is the caller's doing, not a fault of ours,
|
||||
# and it is about to be answered anyway — returning None drops the request into the
|
||||
# counting path, which refuses on the real bytes. Logging it would hand an anonymous
|
||||
# client a free line in our log for every request it cares to malform.
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
async def _send_too_large(send: Send) -> None:
|
||||
await send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": 413,
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"content-length", str(len(TOO_LARGE_BODY)).encode()),
|
||||
],
|
||||
}
|
||||
)
|
||||
await send({"type": "http.response.body", "body": TOO_LARGE_BODY})
|
||||
|
||||
|
||||
class BodyLimitMiddleware:
|
||||
"""Refuses any request body over its route's ceiling, before the application sees it.
|
||||
|
||||
Registered just inside CORS in `main.py` — so the 413 is decorated like every other response
|
||||
and a browser can actually read it — but outside the session, the setup gate and every auth
|
||||
dependency, none of which run for a request that is already too large.
|
||||
|
||||
Every request gets a finite ceiling: `ROUTE_MAX_BODY_BYTES` raises it for the two routes that
|
||||
genuinely take bulk bytes, and there is no path that skips counting altogether.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app: ASGIApp,
|
||||
*,
|
||||
limit: int = DEFAULT_MAX_BODY_BYTES,
|
||||
routes: dict[tuple[str, str], int] | None = None,
|
||||
) -> None:
|
||||
self.app = app
|
||||
self.limit = limit
|
||||
self._routes = tuple(
|
||||
(method, template_to_regex(path), cap)
|
||||
for (method, path), cap in (ROUTE_MAX_BODY_BYTES if routes is None else routes).items()
|
||||
)
|
||||
|
||||
def limit_for(self, method: str, path: str) -> int:
|
||||
"""This request's ceiling: the route's own if it has one, else the global default. Every
|
||||
request gets a finite number — there is no path through here that is simply not counted."""
|
||||
for m, rx, cap in self._routes:
|
||||
if m == method and rx.match(path):
|
||||
return cap
|
||||
return self.limit
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
limit = self.limit_for(scope.get("method", ""), scope.get("path", ""))
|
||||
|
||||
declared = _declared_length(scope)
|
||||
if declared is not None and declared > limit:
|
||||
# Refused on the declaration alone: nothing downstream is called, and not one byte of
|
||||
# the body is read.
|
||||
await _send_too_large(send)
|
||||
return
|
||||
|
||||
seen = 0
|
||||
refused = False
|
||||
started = False
|
||||
|
||||
async def counting_receive() -> Message:
|
||||
nonlocal seen, refused
|
||||
message = await receive()
|
||||
if message["type"] != "http.request":
|
||||
return message
|
||||
seen += len(message.get("body", b""))
|
||||
if seen > limit:
|
||||
# The app is blocked on this receive, so it has not answered yet. Telling it the
|
||||
# client went away is the only in-band way to stop it reading: `request.stream()`
|
||||
# turns an `http.disconnect` into ClientDisconnect, which unwinds the handler. Our
|
||||
# own 413 goes out below, once the app has finished unwinding.
|
||||
refused = True
|
||||
return {"type": "http.disconnect"}
|
||||
return message
|
||||
|
||||
async def guarded_send(message: Message) -> None:
|
||||
nonlocal started
|
||||
if refused and not started:
|
||||
return # discard the app's answer to the disconnect; the 413 below is the response
|
||||
started = True
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, counting_receive, guarded_send)
|
||||
except Exception: # noqa: BLE001 — only swallowed when WE caused it, see below
|
||||
# ClientDisconnect (or whatever a handler raises on a truncated read) is the expected
|
||||
# consequence of the disconnect we just faked, and the caller must get the 413 rather
|
||||
# than a 500. Anything raised when we did NOT refuse is a real failure and is re-raised
|
||||
# untouched.
|
||||
if not refused:
|
||||
raise
|
||||
|
||||
if refused and not started:
|
||||
await _send_too_large(send)
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Alphabetical order in the reader's language (E30 S3).
|
||||
|
||||
Why not the database default: the Postgres image is Alpine, and musl's `en_US.utf8` collation is BYTE
|
||||
order — every accented initial sorted after "Z" ("Zorró < Életrajzi < Ötödik"), in every A→Z list the
|
||||
server orders. Postgres ships the ICU collations on the same image (dev, test, prod alike), so a text
|
||||
sort names one: Hungarian puts "Ö" after "O" and "Cs" after "Cu", English folds the accents in.
|
||||
|
||||
`collated` wraps an ORDER BY expression — and because a keyset cursor compares with the SAME expression
|
||||
(`feedquery.sort_keys`), the cursor pages in the same order it sorted in. `sorted_words` orders a Python
|
||||
list the same way through the database, so a list built in Python matches one built in SQL exactly."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import SQLColumnExpression, column, func, select, values
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
from sqlalchemy.types import String
|
||||
|
||||
from app.mailtext import normalize
|
||||
|
||||
ICU: dict[str, str] = {"en": "en-x-icu", "hu": "hu-x-icu"}
|
||||
|
||||
|
||||
def collation(lang: str | None) -> str:
|
||||
"""The ICU collation for a UI language; anything unknown sorts as English."""
|
||||
return ICU[normalize(lang)]
|
||||
|
||||
|
||||
def collated(expr: SQLColumnExpression[Any], lang: str | None) -> ColumnElement[Any]:
|
||||
"""`expr` compared in the reader's alphabet. Case-folded, so "apple" and "Apple" sit together."""
|
||||
return func.lower(expr).collate(collation(lang))
|
||||
|
||||
|
||||
def sorted_words(db: Session, words: list[str], lang: str | None) -> list[str]:
|
||||
"""`words` in the reader's alphabetical order, by the same ICU collation as `collated`."""
|
||||
if not words:
|
||||
return []
|
||||
w = values(column("w", String), name="words").data([(x,) for x in words])
|
||||
return list(db.scalars(select(w.c.w).order_by(collated(w.c.w, lang), w.c.w)))
|
||||
+96
-7
@@ -1,6 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args, get_origin
|
||||
|
||||
from pydantic import model_validator
|
||||
from pydantic import Field, ValidationInfo, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Shipped placeholder; production must override it. Used by the startup guard below.
|
||||
@@ -15,11 +16,40 @@ def _version_from_file() -> str:
|
||||
v = p.read_text(encoding="utf-8").strip()
|
||||
if v:
|
||||
return v
|
||||
# silent-ok: this runs at import, and `app.logs` imports THIS module — a log call here
|
||||
# would be a circular import. The loop tries the other location, then falls back to "dev".
|
||||
except OSError:
|
||||
continue
|
||||
return "dev"
|
||||
|
||||
|
||||
# Closed value sets. A setting typed with one of these is a CHOICE: pydantic rejects anything else at
|
||||
# boot, and sysconfig derives the admin picker's options from the same Literal — so the list lives in
|
||||
# exactly one place and a consumer never has to guess what a typo should fall back to.
|
||||
SearchSource = Literal["scrape", "api"]
|
||||
DownloadLayout = Literal["plex", "flat"]
|
||||
LogLevel = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR"]
|
||||
LogFormat = Literal["json", "text"]
|
||||
TranscodeHw = Literal["auto", "on", "off"]
|
||||
# libx264's presets, fastest first (`placebo` left out on purpose: no measurable gain for its cost).
|
||||
TranscodePreset = Literal[
|
||||
"ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"
|
||||
]
|
||||
|
||||
|
||||
def choices_of(annotation: object) -> tuple[str, ...]:
|
||||
"""The allowed values of a choice-typed setting, or () for any other type."""
|
||||
return tuple(get_args(annotation)) if get_origin(annotation) is Literal else ()
|
||||
|
||||
|
||||
def canonical_choice(options: tuple[str, ...], raw: object) -> str | None:
|
||||
"""The option `raw` names, in its canonical spelling (any letter case, surrounding space
|
||||
ignored), or None. The ONE matching rule for a choice value from the env and from the admin
|
||||
API, so a spelling one accepts the other cannot refuse."""
|
||||
folded = str(raw).strip().casefold()
|
||||
return next((option for option in options if option.casefold() == folded), None)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
@@ -57,7 +87,7 @@ class Settings(BaseSettings):
|
||||
# VPS Caddy front door. A request from any other peer (someone hitting the app port directly)
|
||||
# is NOT trusted and its raw socket IP is used, so it can't spoof its rate-limit identity via
|
||||
# XFF. Empty = trust no proxy (use the direct peer) — correct for a directly-exposed / local-dev
|
||||
# deploy. Comma-separated. See routes.auth._client_ip + the deploy docs.
|
||||
# deploy. Comma-separated. See ratelimit.client_key + the deploy docs.
|
||||
trusted_proxy_ips: str = ""
|
||||
|
||||
# --- Outbound email (onboarding: access-request + approval notices) ---
|
||||
@@ -85,10 +115,13 @@ class Settings(BaseSettings):
|
||||
search_daily_limit_per_user: int = 70
|
||||
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
|
||||
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
|
||||
search_source: str = "scrape"
|
||||
search_source: SearchSource = "scrape"
|
||||
# Recent-first backfill: how far back to fetch on the first pass per channel.
|
||||
backfill_recent_max_videos: int = 100
|
||||
backfill_recent_max_days: int = 365
|
||||
# A channel whose RSS feed has not answered for this many hours gets its newest uploads read
|
||||
# through the Data API instead (1 unit), at most once per this many hours. 0 = never.
|
||||
rss_fallback_hours: int = 2
|
||||
|
||||
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
|
||||
# this duration (and not livestreams) are probed; longer videos are never Shorts.
|
||||
@@ -155,6 +188,9 @@ class Settings(BaseSettings):
|
||||
# per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
|
||||
# worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
|
||||
# Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
|
||||
# A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients
|
||||
# of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there.
|
||||
# See adapters.YouTubeAdapter._COOKIE_CAPABLE.
|
||||
download_player_clients: str = "android_vr"
|
||||
download_player_clients_fallback: str = "web_safari,web,android"
|
||||
|
||||
@@ -211,7 +247,11 @@ class Settings(BaseSettings):
|
||||
# Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker
|
||||
# finishes), an upload's size is known at ingest, so this bounds a single file directly; the
|
||||
# user's overall byte footprint is still governed by download_default_max_bytes.
|
||||
download_upload_max_bytes: int = 524_288_000 # 500 MiB
|
||||
# ge=1: this number is also the EDGE ceiling (`bodylimit.ROUTE_MAX_BODY_BYTES`), where 0 is a
|
||||
# real limit of zero rather than the route's "no per-file cap" sentinel — an instance set to 0
|
||||
# would refuse every upload, including the admin page's own way back. To lift the cap in
|
||||
# practice, set a large number.
|
||||
download_upload_max_bytes: int = Field(default=524_288_000, ge=1) # 500 MiB
|
||||
# Retention: an asset expires this many days after its last access; the GC job then
|
||||
# deletes it (once no job references it) after an additional grace window.
|
||||
download_retention_days: int = 30
|
||||
@@ -221,10 +261,27 @@ class Settings(BaseSettings):
|
||||
audit_retention_days: int = 180
|
||||
audit_gc_minutes: int = 1440
|
||||
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
||||
download_layout: str = "plex"
|
||||
download_layout: DownloadLayout = "plex"
|
||||
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
||||
sponsorblock_default: bool = False
|
||||
|
||||
# --- Diagnostics / logging (E22) ---
|
||||
# BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at
|
||||
# startup and on change without a restart. This is what a process logs at before it can reach
|
||||
# the database — and the only knob the worker/runner children read, since they are short-lived.
|
||||
# TRACE (below DEBUG) is the "Teljes" diagnostics level; see app/logs.py.
|
||||
log_level: LogLevel = "INFO"
|
||||
# "json" (one object per line — what Loki and the log_events sink ingest) or "text" (a readable
|
||||
# line for a local terminal). Redaction applies to both.
|
||||
log_format: LogFormat = "json"
|
||||
# Also write events to the `log_events` table (app/logsink.py): WARNING and above, the request
|
||||
# lines, and — once they exist — client batches. Off only where no table is wanted (the tests).
|
||||
log_db_sink: bool = True
|
||||
# Days a `log_events` row is kept (the privacy policy promises this), and how often the log_gc job
|
||||
# enforces it — hourly, so a row never outlives the promise by more than an hour.
|
||||
log_retention_days: int = 7
|
||||
log_gc_minutes: int = 60
|
||||
|
||||
# --- Plex integration (optional) ---
|
||||
# The Plex module plays the LOCAL physical media file directly (our own ffmpeg/direct-serve);
|
||||
# Plex is used only for metadata + optional watch-state sync. Requires the Plex server to be
|
||||
@@ -262,16 +319,25 @@ class Settings(BaseSettings):
|
||||
# re-encode is CPU-heavy (the VAAPI hardware path is far cheaper and could sustain more); direct-serve
|
||||
# and remux (stream-copy) are not capped by this.
|
||||
plex_max_transcodes: int = 4
|
||||
# Max concurrent playback sessions of ANY kind — remux (stream-copy) included. Cheap on the CPU but
|
||||
# each one is a whole segmented copy of the film on the scratch volume, written faster than
|
||||
# realtime. One account holds at most two regardless (plex/stream.py `_MAX_SESSIONS_PER_USER`).
|
||||
plex_max_sessions: int = 8
|
||||
# --- Transcode (playable="transcode") re-encode tuning (E6 S1) ---
|
||||
# Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the
|
||||
# source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render
|
||||
# node exists (skip the decodability gate); "off" = always software. See plex/stream.py `_select_encode`.
|
||||
plex_transcode_hw: str = "auto"
|
||||
plex_transcode_hw: TranscodeHw = "auto"
|
||||
# Software-fallback libx264 tuning (IGNORED on the VAAPI path). `preset` trades CPU for size; `crf`
|
||||
# trades quality for size (lower = better + bigger). `max_height` caps the SOFTWARE transcode's output
|
||||
# height (0 = keep source) so a CPU re-encode of a tall source stays affordable — the GPU path always
|
||||
# encodes at source resolution.
|
||||
plex_transcode_preset: str = "veryfast"
|
||||
# Loudness normalisation of the downmixed stereo audio on the HLS path (EBU R128 `loudnorm`).
|
||||
# OFF by default: it makes a film ~11 dB louder and narrows its dynamic range, which is a taste
|
||||
# call, not a fix. Only affects files that go through remux/transcode — a direct-played file is
|
||||
# served untouched, but those are browser-native stereo anyway, not the 5.1 mixes this is for.
|
||||
plex_audio_loudnorm: bool = False
|
||||
plex_transcode_preset: TranscodePreset = "veryfast"
|
||||
plex_transcode_crf: int = 20
|
||||
plex_transcode_max_height: int = 1080
|
||||
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
|
||||
@@ -289,10 +355,22 @@ class Settings(BaseSettings):
|
||||
tmdb_api_base: str = "https://api.themoviedb.org/3"
|
||||
# Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default.
|
||||
tmdb_language: str = "en-US"
|
||||
|
||||
# --- Bug reports (E16): where a debug session's issue goes. All three empty = no tracker; the
|
||||
# composer still exports a zip. The repository must be PRIVATE — an issue carries the reporter's
|
||||
# log — and app/issues/gitea.py refuses a public one. Runtime knobs (sysconfig, admin UI). ---
|
||||
issues_gitea_url: str = ""
|
||||
issues_gitea_repo: str = ""
|
||||
issues_gitea_token: str = ""
|
||||
# How often the TMDB enrichment job fetches a batch of not-yet-enriched Plex titles, in minutes.
|
||||
tmdb_enrich_interval_min: int = 120
|
||||
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||
tmdb_enrich_batch: int = 100
|
||||
# How often the people job (E30 S6) fetches a batch of person bios from TMDB and portraits through Plex,
|
||||
# in minutes, and how many of each per run: ~18k linked people at 300 per 10 min is about ten hours
|
||||
# once, then only the newcomers.
|
||||
people_enrich_interval_min: int = 10
|
||||
people_enrich_batch: int = 300
|
||||
|
||||
@property
|
||||
def allowed_email_set(self) -> set[str]:
|
||||
@@ -320,6 +398,17 @@ class Settings(BaseSettings):
|
||||
https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev."""
|
||||
return self.oauth_redirect_url.lower().startswith("https://")
|
||||
|
||||
@field_validator("*", mode="before")
|
||||
@classmethod
|
||||
def _canonical_choice(cls, value: object, info: ValidationInfo) -> object:
|
||||
"""A choice setting accepts its value in any letter case (`LOG_LEVEL=debug`) and stores the
|
||||
canonical spelling; an unknown value still fails the boot with pydantic's own message."""
|
||||
field = cls.model_fields.get(info.field_name or "")
|
||||
options = choices_of(field.annotation) if field else ()
|
||||
if options and isinstance(value, str):
|
||||
return canonical_choice(options, value) or value
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _enforce_production_secrets(self) -> "Settings":
|
||||
"""Fail fast rather than boot a public instance with the shipped placeholder
|
||||
|
||||
+55
-2
@@ -1,6 +1,6 @@
|
||||
from collections.abc import Iterator
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
from app.config import settings
|
||||
@@ -12,7 +12,9 @@ class Base(DeclarativeBase):
|
||||
|
||||
# Pool sized above the sync-endpoint threadpool (~40) so a burst of short DB sessions can't starve on
|
||||
# checkout. The image proxies deliberately DON'T hold a connection during their slow external fetch
|
||||
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones.
|
||||
# (see routes/plex/), so this headroom is only for the many brief sessions, not long ones. The one
|
||||
# long hold — a playlist sync's advisory claim, for a whole push — opens its own unpooled connection
|
||||
# (sync/playlists.py `_claim_engine`) instead of drawing a third one from here.
|
||||
engine = create_engine(
|
||||
settings.database_url,
|
||||
pool_pre_ping=True,
|
||||
@@ -20,6 +22,10 @@ engine = create_engine(
|
||||
max_overflow=30,
|
||||
pool_timeout=30,
|
||||
future=True,
|
||||
# A database error renders its bound parameters into its message — a title, an uploader, a
|
||||
# file path, a URL — and that message is logged and stored as a job's error text. The SQL
|
||||
# stays; the values are data, and the ids an operator needs are on the line already.
|
||||
hide_parameters=True,
|
||||
)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
|
||||
|
||||
@@ -30,3 +36,50 @@ def get_db() -> Iterator[Session]:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def release_read_connection(db: Session) -> None:
|
||||
"""End a request's READ transaction so its pooled connection goes back BEFORE a long await.
|
||||
|
||||
SQLAlchemy 2.0 autobegins on the first SELECT, so a request that has merely authenticated is
|
||||
already holding one of the connections above — `idle in transaction` to Postgres, and nothing
|
||||
times it out. A route that then awaits something slow (a request body still arriving, an
|
||||
external fetch) pins it for the whole wait, which is precisely the invariant the pool is sized
|
||||
on in the comment above. Measured: the next checkout after this call reuses the very backend
|
||||
that was released, and a 1.8 s upload goes from one sustained `idle in transaction` to none.
|
||||
|
||||
Every route that reads its own body from a dependency needs this, and `test_body_limit.py`
|
||||
holds that — the three that do are the ones with a raised (or self-imposed) body ceiling, and
|
||||
they got here one at a time, which is how the upload route ended up the only one fixed.
|
||||
|
||||
The caller must have NOTHING PENDING, and that is checked rather than described. The session is
|
||||
shared with every dependency that ran before, so a rollback here would silently discard a write
|
||||
one of them staged — `service.enqueue` already learned that lesson the expensive way and uses a
|
||||
SAVEPOINT for it. CLAUDE.md: a rule stated in a comment must also be enforced in code.
|
||||
|
||||
TWO checks, because the obvious one is blind to the shapes this codebase actually writes in.
|
||||
`db.new`/`deleted`/`dirty` hold only UNFLUSHED unit-of-work: the moment a helper calls
|
||||
`db.flush()` — CLAUDE.md's canonical split is "the handler commits; helpers flush" — the object
|
||||
is persistent with no unflushed history and all three sets are empty again, and Core DML
|
||||
(`db.execute(update(...))`) never enters them at all. Measured: after `add`+`flush` the sets are
|
||||
empty, the old check passed, and the rollback lost the row. So the transaction is asked
|
||||
directly. Postgres assigns a transaction id on the first WRITE and never for a read, so
|
||||
`pg_current_xact_id_if_assigned()` is NULL for exactly the state this function is safe in — and
|
||||
it sees a raw `text("UPDATE …")` too, which no session-level bookkeeping would.
|
||||
"""
|
||||
if db.new or db.deleted or db.dirty:
|
||||
raise RuntimeError(
|
||||
"release_read_connection() with pending work: something ahead of this dependency "
|
||||
"staged a write, which ending the transaction would discard. Commit it first, or do "
|
||||
"it after the await."
|
||||
)
|
||||
if (
|
||||
db.in_transaction()
|
||||
and db.execute(text("SELECT pg_current_xact_id_if_assigned()")).scalar() is not None
|
||||
):
|
||||
raise RuntimeError(
|
||||
"release_read_connection() on a transaction that has already WRITTEN: something ahead "
|
||||
"of this dependency flushed a row or ran an UPDATE/INSERT/DELETE, which ending the "
|
||||
"transaction would discard. Commit it first, or do it after the await."
|
||||
)
|
||||
db.rollback()
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, and the evidence a bug report is built from.
|
||||
|
||||
While a session is open its user's browser names it in `X-Debug-Session` on every request. The auth
|
||||
dependency accepts the header only for the caller's OWN open session (anything else is ignored, never
|
||||
an error — a stale tab must not break the app) and then logs that request at the session's level with
|
||||
every line tagged by its id; the browser tags its own events the same way. The issue is later built
|
||||
from exactly the rows carrying the tag, so what the review list shows is what gets sent.
|
||||
|
||||
Starting a session IS the consent for its level, for its duration: the start dialog carries the
|
||||
consent text. It needs no change to the user's own diagnostics level, and none is made — when the
|
||||
session ends the level in force is simply the user's own again.
|
||||
|
||||
A session is capped at `MAX_MINUTES`, set by the server at Start. An open one past its cap is over
|
||||
whether or not anything has closed it yet: it reads as over at once, a new Start closes it, and the
|
||||
hourly log job sweeps the rest. Sessions go with the logs they point at, after `log_retention_days`.
|
||||
|
||||
Core layer: imports `models`, `diagnostics`.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import StrEnum
|
||||
from typing import Literal, get_args
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import and_, delete, func, not_, select, update
|
||||
from sqlalchemy.orm import Session, defer
|
||||
|
||||
from app import diagnostics
|
||||
from app.models import DebugScreenshot, DebugSession, LogEvent, User
|
||||
|
||||
HEADER = "x-debug-session"
|
||||
MAX_MINUTES = diagnostics.SESSION_MINUTES
|
||||
SCREENSHOT_MAX_BYTES = 10 * 1024 * 1024
|
||||
SCREENSHOTS_MAX = 8
|
||||
# What one user may keep in screenshots across ALL their sessions (until retention takes them): the
|
||||
# per-session cap alone would let Start → 8 images → Stop repeat without end.
|
||||
SCREENSHOT_USER_MAX_BYTES = 100 * 1024 * 1024
|
||||
# The screenshot types kept, and the extension each is exported and sent under — one table, so an
|
||||
# accepted type always has a name.
|
||||
IMAGE_TYPES = {"image/png": "png", "image/jpeg": "jpg", "image/webp": "webp"}
|
||||
# What a screenshot is called in the zip and on the issue. `issues/bundle.py` builds the names from
|
||||
# this and `IMAGE_TYPES`; this module and `issues/gitea.py` read them back to tell a screenshot's
|
||||
# link from a log's. One definition, because three copies of a name is how the next rename breaks
|
||||
# `images_pending` into silence.
|
||||
SCREENSHOT_PREFIX = "screenshot-"
|
||||
# Rows an issue can carry. A 60-minute session at Teljes is a few thousand rows; past this the
|
||||
# bundle says how many it left out rather than growing without bound.
|
||||
BUNDLE_MAX = 5_000
|
||||
|
||||
SessionLevel = Literal["DEBUG", "TRACE"]
|
||||
SESSION_LEVELS: tuple[str, ...] = get_args(SessionLevel)
|
||||
# Detail order of the diagnostics levels: a session raises the level in force, never lowers it.
|
||||
_DETAIL = {"INFO": 0, "DEBUG": 1, "TRACE": 2}
|
||||
_ID_LEN = 16
|
||||
|
||||
|
||||
class EndReason(StrEnum):
|
||||
STOPPED = "stopped"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
def is_live(session: DebugSession, now: datetime | None = None) -> bool:
|
||||
"""Open and within its cap — THE liveness rule: an open session past its cap is over."""
|
||||
return session.ended_at is None and session.until > (now or datetime.now(timezone.utc))
|
||||
|
||||
|
||||
def open_session(db: Session, user: User) -> DebugSession | None:
|
||||
"""The user's live session, or None. Read-only: one past its cap is over and reads as None;
|
||||
`start` and the hourly sweep are what close it."""
|
||||
session = _unclosed(db, user)
|
||||
if session is None or not is_live(session, datetime.now(timezone.utc)):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def _unclosed(db: Session, user: User) -> DebugSession | None:
|
||||
return db.scalar(
|
||||
select(DebugSession).where(DebugSession.user_id == user.id, DebugSession.ended_at.is_(None))
|
||||
)
|
||||
|
||||
|
||||
def start(db: Session, user: User, level: SessionLevel) -> DebugSession:
|
||||
"""Open a session, or return the live one — a second tab pressing Start joins the recording
|
||||
rather than ending it. One past its cap is closed first: the one-open-per-user index would
|
||||
refuse the new row beside it. Flushes; the caller commits. Two concurrent Starts can still both
|
||||
get here: the index refuses the second, and the caller joins the one that won."""
|
||||
now = datetime.now(timezone.utc)
|
||||
current = _unclosed(db, user)
|
||||
if current is not None:
|
||||
if is_live(current, now):
|
||||
return current
|
||||
end(current, EndReason.EXPIRED)
|
||||
db.flush()
|
||||
session = DebugSession(
|
||||
id=secrets.token_hex(_ID_LEN // 2),
|
||||
user_id=user.id,
|
||||
level=level,
|
||||
started_at=now,
|
||||
until=now + timedelta(minutes=MAX_MINUTES),
|
||||
)
|
||||
db.add(session)
|
||||
db.flush()
|
||||
return session
|
||||
|
||||
|
||||
def stop(session: DebugSession) -> None:
|
||||
"""The user's Stop. Past its cap the session had already ended on its own, and a late Stop must
|
||||
not rename that."""
|
||||
end(session, EndReason.STOPPED if is_live(session) else EndReason.EXPIRED)
|
||||
|
||||
|
||||
def end(session: DebugSession, reason: EndReason) -> None:
|
||||
"""Close a session. An expired one is dated at its cap, not at whenever it was noticed."""
|
||||
if session.ended_at is not None:
|
||||
return
|
||||
session.ended_at = (
|
||||
session.until
|
||||
if reason is EndReason.EXPIRED
|
||||
else min(datetime.now(timezone.utc), session.until)
|
||||
)
|
||||
session.ended_reason = reason
|
||||
|
||||
|
||||
def from_header(db: Session, user: User, value: str | None) -> DebugSession | None:
|
||||
"""The session a request names, if it is the caller's own and still live; else None."""
|
||||
if not value or len(value) != _ID_LEN or not all(c in "0123456789abcdef" for c in value):
|
||||
return None
|
||||
session = db.get(DebugSession, value)
|
||||
if (
|
||||
session is None
|
||||
or session.user_id != user.id
|
||||
or not is_live(session, datetime.now(timezone.utc))
|
||||
):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def level_for(user: User, session: DebugSession | None) -> tuple[str, datetime | None]:
|
||||
"""The diagnostics level (and its expiry) a request is logged at: the user's own, raised to the
|
||||
session's while one is named — whichever records more.
|
||||
|
||||
A named session also caps the expiry at its own: this pair is what the process tracks under the
|
||||
session (app/logs.py), and an expiry outliving the recording would hold the root logger down
|
||||
after it ended."""
|
||||
own = diagnostics.effective_level(user)
|
||||
until = user.diagnostics_until if own != diagnostics.BASE else None
|
||||
if session is None:
|
||||
return own, until
|
||||
if _DETAIL[session.level] <= _DETAIL[own]:
|
||||
return own, min(until, session.until) if until else session.until
|
||||
return session.level, session.until
|
||||
|
||||
|
||||
def lock(db: Session, session: DebugSession) -> None:
|
||||
"""Re-read the session under a row lock, held until the caller commits. Everything that changes
|
||||
what an issue is made of — the draft, the screenshots, the send itself — takes it first: a send
|
||||
in flight sees none of them change under it, and each of them sees a send that committed."""
|
||||
db.refresh(session, with_for_update=True)
|
||||
|
||||
|
||||
def save_draft(session: DebugSession, draft: dict) -> None:
|
||||
"""Keep the composer's draft (the caller validated, locked and commits). Shape: `title`, `body`
|
||||
(the rendered markdown an export or an issue carries), `excluded` (log row ids taken out of the
|
||||
evidence) and `form` (the composer's own fields, to reopen it as it was). A delivery's progress
|
||||
is not the composer's to replace, and is kept."""
|
||||
progress = (session.draft or {}).get("delivery")
|
||||
session.draft = {**draft, "delivery": progress} if progress else draft
|
||||
|
||||
|
||||
# --- a delivery's progress -----------------------------------------------------------------
|
||||
# Written by app/issues/gitea.py into the draft; read HERE, so the send's decision and what the
|
||||
# composer shows are one reading of it.
|
||||
def delivery(session: DebugSession) -> dict:
|
||||
return dict((session.draft or {}).get("delivery") or {})
|
||||
|
||||
|
||||
def not_uploaded(session: DebugSession) -> list[str]:
|
||||
"""The files a send that stopped half-way has still to upload — by name, as the issue carries
|
||||
them. What the composer names to the user."""
|
||||
return list(delivery(session).get("pending") or [])
|
||||
|
||||
|
||||
def images_pending(session: DebugSession) -> bool:
|
||||
"""Whether the issue's body still has to be edited to show the screenshots it already carries
|
||||
(every upload went, the edit that shows them did not)."""
|
||||
state = delivery(session)
|
||||
shown = state.get("images_shown")
|
||||
return not shown and any(n.startswith(SCREENSHOT_PREFIX) for n in state.get("links") or {})
|
||||
|
||||
|
||||
def unfinished(session: DebugSession) -> bool:
|
||||
"""Whether a send owes the issue anything — the retry's question."""
|
||||
return bool(not_uploaded(session)) or images_pending(session)
|
||||
|
||||
|
||||
# A send claims the session while it talks to the tracker, instead of holding the row's lock across
|
||||
# those calls: a Stop, an autosave or a screenshot would queue behind a lock for the whole send.
|
||||
# They see the claim and are refused at once instead. A claim not renewed for this long was left by
|
||||
# a request that died mid-send (the process restarted), and the next send takes it over — which is
|
||||
# why a live send RENEWS it every time it reports progress (`set_delivery`). A claim is a lease, and
|
||||
# a send slower than the lease must not lose it while it is still running.
|
||||
SEND_CLAIM_SECONDS = 600
|
||||
|
||||
|
||||
def _claim(session: DebugSession) -> dict:
|
||||
claim = delivery(session).get("sending")
|
||||
return claim if isinstance(claim, dict) else {}
|
||||
|
||||
|
||||
def _stamp(token: str) -> dict:
|
||||
return {"at": datetime.now(timezone.utc).isoformat(), "by": token}
|
||||
|
||||
|
||||
def sending(session: DebugSession) -> bool:
|
||||
"""Whether a send is under way right now (a live claim)."""
|
||||
at = _claim(session).get("at")
|
||||
if not at:
|
||||
return False
|
||||
return (datetime.now(timezone.utc) - datetime.fromisoformat(at)).total_seconds() < (
|
||||
SEND_CLAIM_SECONDS
|
||||
)
|
||||
|
||||
|
||||
def claim_send(session: DebugSession) -> str:
|
||||
"""Mark the session as being sent, and return the token that OWNS this claim. The caller holds
|
||||
the lock and commits, which is what makes the claim visible to everyone else."""
|
||||
token = uuid4().hex
|
||||
_write_delivery(session, {**delivery(session), "sending": _stamp(token)})
|
||||
return token
|
||||
|
||||
|
||||
def release_send(session: DebugSession, token: str) -> None:
|
||||
"""Give back our OWN claim, and only ours: one that aged out and was taken over belongs to the
|
||||
send holding it now, and handing that one back would let a third send in beside it."""
|
||||
if _claim(session).get("by") != token:
|
||||
return
|
||||
_write_delivery(session, {k: v for k, v in delivery(session).items() if k != "sending"})
|
||||
|
||||
|
||||
class SendClaimLost(Exception):
|
||||
"""This send's claim is no longer on the row: it aged out and another send took it over. The
|
||||
send must stop — what it uploads from here it can no longer record, so a retry would send it
|
||||
again — and it must not write, because the row now holds the other send's progress."""
|
||||
|
||||
|
||||
def claim_owner(db: Session, session: DebugSession) -> str | None:
|
||||
"""Who holds the row's claim RIGHT NOW, read from the ROW and not from the object.
|
||||
|
||||
`expire_on_commit` is off (app/db.py), so a checkpoint does not re-read: a send that stalled
|
||||
past its lease still carries its own dead token in memory long after another send took over.
|
||||
A targeted scalar, not `lock`/`db.refresh` — a refresh here would discard the `issue_url` that
|
||||
`gitea.deliver` assigns one line before its first progress report, and that link is the one
|
||||
thing the send must not lose."""
|
||||
return db.scalar(
|
||||
select(DebugSession.draft["delivery"]["sending"]["by"].as_string()).where(
|
||||
DebugSession.id == session.id
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def set_delivery(db: Session, session: DebugSession, state: dict, token: str) -> None:
|
||||
"""Write a delivery's progress (app/issues/gitea.py keeps it; this module reads it), and renew
|
||||
the claim while doing it — a send still reporting progress is still alive, so one slower than
|
||||
the lease cannot expire while it is still running.
|
||||
|
||||
The claim is not the caller's to carry: `gitea.deliver` threads ONE snapshot of this dict
|
||||
through a whole send, so a progress report that carried the claim wrote a stale one back at
|
||||
every step. It is stamped from `token` here instead, and only after the ROW confirms the claim
|
||||
is still ours. Without that check the write is a full-column overwrite with no WHERE: a send
|
||||
that lost its lease came back and stomped the take-over send's committed links, then put its
|
||||
own dead token back on an idle session (deep-review round 4). `release_send` has always made
|
||||
the same check; this is the other half of it."""
|
||||
if claim_owner(db, session) != token:
|
||||
raise SendClaimLost(session.id)
|
||||
state = {k: v for k, v in state.items() if k != "sending"}
|
||||
state["sending"] = _stamp(token)
|
||||
_write_delivery(session, state)
|
||||
|
||||
|
||||
def _write_delivery(session: DebugSession, state: dict) -> None:
|
||||
# A new dict, so the JSONB column sees the change.
|
||||
session.draft = {**(session.draft or {}), "delivery": state}
|
||||
|
||||
|
||||
def serialize(session: DebugSession | None, screenshots: int = 0) -> dict | None:
|
||||
if session is None:
|
||||
return None
|
||||
return {
|
||||
"id": session.id,
|
||||
"level": session.level,
|
||||
"started_at": session.started_at.isoformat(),
|
||||
"until": session.until.isoformat(),
|
||||
"ended_at": session.ended_at.isoformat() if session.ended_at else None,
|
||||
"ended_reason": session.ended_reason,
|
||||
"issue_url": session.issue_url,
|
||||
# A send that stopped half-way: the files it has still to upload, and whether the issue's
|
||||
# body has still to be edited to show the screenshots it already carries. Both are true of a
|
||||
# send that is RUNNING too — every step commits its progress — so `sending` goes with them:
|
||||
# without it a reload or a second tab renders a live send as a failed one, and offers a
|
||||
# retry that is certain to be refused.
|
||||
"pending_files": not_uploaded(session),
|
||||
"images_pending": images_pending(session),
|
||||
"sending": sending(session),
|
||||
"screenshots": screenshots,
|
||||
"draft": session.draft,
|
||||
}
|
||||
|
||||
|
||||
def view(db: Session, session: DebugSession) -> dict:
|
||||
"""The session as the API serves it — one shape wherever it is served."""
|
||||
return serialize(session, screenshot_count(db, session)) or {}
|
||||
|
||||
|
||||
def screenshot_count(db: Session, session: DebugSession) -> int:
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.count())
|
||||
.select_from(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
# --- screenshots ---------------------------------------------------------------------------
|
||||
# By content, not by the declared type: what is stored is served back to a browser as an image.
|
||||
_SIGNATURES = (
|
||||
(b"\x89PNG\r\n\x1a\n", "image/png"),
|
||||
(b"\xff\xd8\xff", "image/jpeg"),
|
||||
)
|
||||
|
||||
|
||||
def sniff(data: bytes) -> str | None:
|
||||
"""The image type `data` really is — one of IMAGE_TYPES — or None."""
|
||||
for magic, mime in _SIGNATURES:
|
||||
if data.startswith(magic):
|
||||
return mime
|
||||
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
return None
|
||||
|
||||
|
||||
def lock_user(db: Session, user_id: int) -> None:
|
||||
"""Hold this user's own lock for the rest of the transaction. A session's row lock bounds what
|
||||
THAT session may keep; the byte cap below is the user's across all of them, and a lock on one
|
||||
session cannot serialize a check that reads them all — two uploads to two sessions both read the
|
||||
total before either commits, and both pass."""
|
||||
db.execute(select(func.pg_advisory_xact_lock(user_id)))
|
||||
|
||||
|
||||
def stored_screenshot_bytes(db: Session, user_id: int) -> int:
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.sum(DebugScreenshot.size), 0))
|
||||
.join(DebugSession, DebugSession.id == DebugScreenshot.session_id)
|
||||
.where(DebugSession.user_id == user_id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
class ScreenshotRefused(Exception):
|
||||
"""A screenshot that cannot be kept; `code` is the refusal's machine word."""
|
||||
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def add_screenshot(db: Session, session: DebugSession, data: bytes) -> DebugScreenshot:
|
||||
"""Keep one screenshot. Locks the user and the session, and flushes; the caller commits.
|
||||
Refused while a send is under way or once the issue was sent (it can no longer carry it), past
|
||||
the per-session count or the user's storage, or when it is not an image we can show."""
|
||||
if not data:
|
||||
raise ScreenshotRefused("empty")
|
||||
mime = sniff(data)
|
||||
if mime is None:
|
||||
raise ScreenshotRefused("not_an_image")
|
||||
# Under both locks: two uploads at once cannot both pass the count (the session's), nor both
|
||||
# pass the user's byte cap from two different sessions (the user's). Always in this order.
|
||||
lock_user(db, session.user_id)
|
||||
lock(db, session)
|
||||
if sending(session):
|
||||
raise ScreenshotRefused("sending")
|
||||
if session.issue_url is not None:
|
||||
raise ScreenshotRefused("already_sent")
|
||||
if screenshot_count(db, session) >= SCREENSHOTS_MAX:
|
||||
raise ScreenshotRefused("too_many")
|
||||
if stored_screenshot_bytes(db, session.user_id) + len(data) > SCREENSHOT_USER_MAX_BYTES:
|
||||
raise ScreenshotRefused("storage_full")
|
||||
shot = DebugScreenshot(
|
||||
session_id=session.id,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
mime=mime,
|
||||
size=len(data),
|
||||
data=data,
|
||||
)
|
||||
db.add(shot)
|
||||
db.flush()
|
||||
return shot
|
||||
|
||||
|
||||
def remove_screenshot(db: Session, session: DebugSession, shot: DebugScreenshot) -> None:
|
||||
"""Delete one screenshot, under the lock. Refused while a send is under way or unfinished: a
|
||||
delivery names the screenshots by their order, and removing one would renumber what it still
|
||||
has to send. Once the issue has everything, this only drops OUR copy — the issue keeps its
|
||||
own. The caller commits."""
|
||||
lock(db, session)
|
||||
if sending(session):
|
||||
raise ScreenshotRefused("sending")
|
||||
if unfinished(session):
|
||||
raise ScreenshotRefused("already_sent")
|
||||
db.delete(shot)
|
||||
db.flush()
|
||||
|
||||
|
||||
def screenshots(db: Session, session: DebugSession) -> list[DebugScreenshot]:
|
||||
"""The session's screenshots, oldest first, WITHOUT their bytes loaded."""
|
||||
return list(
|
||||
db.scalars(
|
||||
select(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
.options(defer(DebugScreenshot.data))
|
||||
.order_by(DebugScreenshot.id)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --- the evidence --------------------------------------------------------------------------
|
||||
# The browser's own log upload (`POST` here, every few seconds while recording) is how the evidence
|
||||
# arrives, not part of it: its server lines would fill the bundle with the recording's plumbing. The
|
||||
# browser leaves its copies out for the same reason (frontend lib/diagnostics `isOwnRequest`).
|
||||
INGEST_PATH = "/api/diagnostics/events"
|
||||
|
||||
|
||||
def _evidence(session: DebugSession):
|
||||
upload_line = and_(
|
||||
LogEvent.event == "http.request", LogEvent.attrs["path"].astext == INGEST_PATH
|
||||
)
|
||||
# `IS TRUE`, not the bare condition: for a row with no event or no attrs it is NULL, and
|
||||
# `NOT NULL` would drop exactly the rows that are evidence.
|
||||
return (LogEvent.debug_session_id == session.id, not_(upload_line.is_(True)))
|
||||
|
||||
|
||||
def bundle(
|
||||
db: Session, session: DebugSession, *, drop_excluded: bool = False
|
||||
) -> tuple[list[LogEvent], int]:
|
||||
"""The session's log rows, browser and server interleaved in time, capped at BUNDLE_MAX — and
|
||||
how many rows it has in all, so a cut bundle can say so. `drop_excluded` leaves out the rows
|
||||
the user took out in the composer: what an export or an issue carries."""
|
||||
where = list(_evidence(session))
|
||||
excluded = (session.draft or {}).get("excluded") or [] if drop_excluded else []
|
||||
if excluded:
|
||||
where.append(LogEvent.id.not_in(excluded))
|
||||
rows = list(
|
||||
db.scalars(
|
||||
select(LogEvent).where(*where).order_by(LogEvent.ts, LogEvent.id).limit(BUNDLE_MAX)
|
||||
)
|
||||
)
|
||||
total = len(rows)
|
||||
if total == BUNDLE_MAX:
|
||||
total = db.scalar(select(func.count()).select_from(LogEvent).where(*where)) or total
|
||||
return rows, total
|
||||
|
||||
|
||||
# --- housekeeping (the hourly log job) -----------------------------------------------------
|
||||
def sweep(db: Session) -> int:
|
||||
"""Stage closing every open session past its cap; returns how many."""
|
||||
now = datetime.now(timezone.utc)
|
||||
result = db.execute(
|
||||
update(DebugSession)
|
||||
.where(DebugSession.ended_at.is_(None), DebugSession.until <= now)
|
||||
.values(ended_at=DebugSession.until, ended_reason=EndReason.EXPIRED.value)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Stage deleting sessions (and by cascade their screenshots) started before the log retention
|
||||
window — the rows they would bundle are gone by then too. Returns how many."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, retention_days))
|
||||
result = db.execute(delete(DebugSession).where(DebugSession.started_at < cutoff))
|
||||
return result.rowcount or 0
|
||||
@@ -0,0 +1,65 @@
|
||||
"""A user's diagnostics level (E22 S5): what they consented to, and for how long.
|
||||
|
||||
Three levels, named in the UI Alap / Részletes / Teljes:
|
||||
|
||||
* INFO — the base, for everyone, no consent. The browser keeps its log in memory; nothing extra
|
||||
is sent or stored.
|
||||
* DEBUG — consented. The server logs this user's requests in detail and stores every line of
|
||||
them, and the browser streams its own log to `POST /api/diagnostics/events`.
|
||||
* TRACE — consented. As DEBUG, plus what the user typed into a search.
|
||||
|
||||
A raised level always lapses after `ELEVATION_HOURS`: the expiry is computed HERE at consent time,
|
||||
never taken from the client, and an expired level reads as INFO without anything having to clear it.
|
||||
The shared demo account cannot raise its level — its log would mix every visitor's session.
|
||||
|
||||
Core layer: imports `models` and `sysconfig`.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Literal, get_args
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.models import User
|
||||
|
||||
DiagnosticsLevel = Literal["INFO", "DEBUG", "TRACE"]
|
||||
LEVELS: tuple[str, ...] = get_args(DiagnosticsLevel)
|
||||
BASE: DiagnosticsLevel = "INFO"
|
||||
ELEVATION_HOURS = 24
|
||||
# How long a debug session records at most (E16, app/debugsession.py). Here, beside the other
|
||||
# consent window, because the start dialog's consent text states it and reads it from `state`.
|
||||
SESSION_MINUTES = 60
|
||||
|
||||
|
||||
def effective_level(user: User, now: datetime | None = None) -> DiagnosticsLevel:
|
||||
"""The level in force for `user` right now: the consented one until it expires, else BASE."""
|
||||
level, until = user.diagnostics_level, user.diagnostics_until
|
||||
if level not in LEVELS or level == BASE or until is None:
|
||||
return BASE
|
||||
if until <= (now or datetime.now(timezone.utc)):
|
||||
return BASE
|
||||
return level # type: ignore[return-value] — narrowed by the LEVELS check above
|
||||
|
||||
|
||||
def state(db: Session, user: User) -> dict:
|
||||
"""The `/api/me` view of it: the level in force, when a raised one lapses, and how long stored
|
||||
log lines are kept — the consent text states that number, so it must be the instance's own."""
|
||||
level = effective_level(user)
|
||||
return {
|
||||
"level": level,
|
||||
"until": user.diagnostics_until.isoformat() if level != BASE else None,
|
||||
"retention_days": sysconfig.get_int(db, "log_retention_days"),
|
||||
"session_minutes": SESSION_MINUTES,
|
||||
}
|
||||
|
||||
|
||||
def set_level(user: User, level: DiagnosticsLevel) -> None:
|
||||
"""Record a consent (or its withdrawal). The caller commits. Raising the level again restarts
|
||||
the clock — each consent is for the next ELEVATION_HOURS, not an extension of an older one."""
|
||||
if level == BASE:
|
||||
user.diagnostics_level = None
|
||||
user.diagnostics_until = None
|
||||
return
|
||||
user.diagnostics_level = level
|
||||
user.diagnostics_until = datetime.now(timezone.utc) + timedelta(hours=ELEVATION_HOURS)
|
||||
@@ -48,6 +48,7 @@ def pick(
|
||||
shelf: list[dict] | None = None,
|
||||
exclude: set[str] | None = None,
|
||||
start: int = 0,
|
||||
lang: str,
|
||||
) -> dict | None:
|
||||
"""`{"tier": ..., "card": ...}`, or None when nothing at all fits.
|
||||
|
||||
@@ -70,8 +71,8 @@ def pick(
|
||||
skip = exclude or set()
|
||||
|
||||
def _resume() -> dict | None:
|
||||
for card in rails.continue_watching(db, user) if resume is None else resume:
|
||||
playable = rails.playable(db, user.id, card)
|
||||
for card in rails.continue_watching(db, user, lang=lang) if resume is None else resume:
|
||||
playable = rails.playable(db, user.id, card, lang=lang)
|
||||
# Excluded on the card the USER saw (the show), not only on what it resolves to, so a
|
||||
# rerolled-past show does not come straight back as its own next episode.
|
||||
if cards.key(card) in skip or (playable is not None and cards.key(playable) in skip):
|
||||
@@ -81,7 +82,11 @@ def pick(
|
||||
return None
|
||||
|
||||
def _comfort() -> dict | None:
|
||||
for card in pins.playables(pins.shelf(db, user) if shelf is None else shelf):
|
||||
if shelf is None:
|
||||
own = pins.shelf(db, user, include_plex=rails.plex_available(db), lang=lang)
|
||||
else:
|
||||
own = shelf
|
||||
for card in pins.playables(own):
|
||||
if cards.key(card) not in skip and fits(card["remaining_seconds"], budget):
|
||||
return card
|
||||
return None
|
||||
@@ -89,7 +94,7 @@ def pick(
|
||||
def _surprise() -> dict | None:
|
||||
# A wider draw than the one card we need: the shuffle is seeded, so without the headroom an
|
||||
# excluded pick would leave the tier empty rather than move past it.
|
||||
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW):
|
||||
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW, lang=lang):
|
||||
if cards.key(card) not in skip:
|
||||
return card
|
||||
return None
|
||||
|
||||
+112
-18
@@ -8,14 +8,16 @@ second one.
|
||||
|
||||
Every shelf card carries a resolved `next`: the thing that actually plays when you click the tile —
|
||||
a show's in-progress-or-next episode, a channel's newest unwatched video, a playlist's first
|
||||
unwatched item. It costs a query per non-leaf pin, which is affordable precisely because this shelf is
|
||||
short and hand-curated, and it buys the property the whole surface exists for: one click, no
|
||||
intermediate page, no second decision.
|
||||
unwatched item. That buys the property the whole surface exists for: one click, no intermediate page,
|
||||
no second decision. Channels and playlists are resolved for the whole shelf at once (`_Batch`), so the
|
||||
cost does not grow a query or two per pin.
|
||||
|
||||
Domain module, so nothing here raises HTTP. `resolve_ref` returns None for "no such thing, or not
|
||||
yours" and the route turns that into its 404 — core never imports routes.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -62,6 +64,8 @@ def resolve_ref(db: Session, user: User, kind: str, ref: str) -> tuple[str, obje
|
||||
return (col, sh.id) if sh is not None else None
|
||||
try:
|
||||
pid = int(ref)
|
||||
# silent-ok: validating a reference the client supplied. "Not an integer" is answered as "no
|
||||
# such pin target", which is the same answer this function gives for one that does not exist.
|
||||
except ValueError:
|
||||
return None
|
||||
pl = db.get(Playlist, pid)
|
||||
@@ -71,6 +75,44 @@ def resolve_ref(db: Session, user: User, kind: str, ref: str) -> tuple[str, obje
|
||||
return (col, pl.id) if pl is not None and pl.user_id == user.id else None
|
||||
|
||||
|
||||
def reorder(db: Session, user: User, ids: list[int]) -> None:
|
||||
"""Put the listed pins in the listed order, WITHIN the slots they already hold.
|
||||
|
||||
The client sends the order it can SEE, which is not always the whole shelf: with Plex off the Home
|
||||
rail hides the Plex pins, and a stale tab may not know a pin added elsewhere. Numbering only the
|
||||
listed ids 0..n-1 collided with every pin left out — two tiles on one position, and the hidden
|
||||
ones jumped about when they came back. So the pins left out keep their slots, and the listed ones
|
||||
are dealt into the slots the listed set occupies. Ids that are not this user's, or repeated, are
|
||||
ignored rather than failing a stale replay wholesale. Flushes; the route commits."""
|
||||
# Locked: the rows it leaves alone are part of the answer too. Two overlapping reorders each
|
||||
# skipped the rows that already matched in their own snapshot and interleaved into two pins on
|
||||
# one slot; the second now waits and deals from the first one's result. Locked in id order and
|
||||
# sorted HERE: with `ORDER BY position … FOR UPDATE` Postgres sorts before it locks, so the one
|
||||
# that waited got the fresh rows in its stale order and dealt the pins it left out back to their
|
||||
# old slots — and one global lock order means two reorders cannot deadlock.
|
||||
locked = db.execute(
|
||||
select(ComfortPin)
|
||||
.where(ComfortPin.user_id == user.id)
|
||||
.order_by(ComfortPin.id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalars()
|
||||
current = sorted(locked, key=lambda p: (p.position, p.id))
|
||||
owned = {p.id for p in current}
|
||||
listed: list[int] = []
|
||||
for i in ids:
|
||||
if i in owned and i not in listed:
|
||||
listed.append(i)
|
||||
chosen = set(listed)
|
||||
queue = iter(listed)
|
||||
order = [next(queue) if p.id in chosen else p.id for p in current]
|
||||
by_id = {p.id: p for p in current}
|
||||
for position, pin_id in enumerate(order):
|
||||
if by_id[pin_id].position != position:
|
||||
by_id[pin_id].position = position
|
||||
db.flush()
|
||||
|
||||
|
||||
def list_pins(db: Session, user: User) -> list[ComfortPin]:
|
||||
return list(
|
||||
db.execute(
|
||||
@@ -101,7 +143,34 @@ def next_position(db: Session, user: User) -> int:
|
||||
)
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
@dataclass
|
||||
class _Batch:
|
||||
"""What the shelf resolves for every pinned channel and playlist in one go."""
|
||||
|
||||
channels: dict[str, Channel] = field(default_factory=dict)
|
||||
channel_summary: dict[str, tuple[int, dict | None]] = field(default_factory=dict)
|
||||
playlists: dict[int, Playlist] = field(default_factory=dict)
|
||||
playlist_summary: dict[int, tuple[int, dict | None]] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def load(cls, db: Session, user: User, pins: list[ComfortPin]) -> "_Batch":
|
||||
ch_ids = [p.channel_id for p in pins if p.channel_id is not None]
|
||||
pl_ids = [p.playlist_id for p in pins if p.playlist_id is not None]
|
||||
b = cls()
|
||||
if ch_ids:
|
||||
b.channels = {
|
||||
c.id: c for c in db.scalars(select(Channel).where(Channel.id.in_(ch_ids)))
|
||||
}
|
||||
b.channel_summary = queries.channels_summary(db, user, ch_ids)
|
||||
if pl_ids:
|
||||
b.playlists = {
|
||||
p.id: p for p in db.scalars(select(Playlist).where(Playlist.id.in_(pl_ids)))
|
||||
}
|
||||
b.playlist_summary = queries.playlists_summary(db, user, pl_ids)
|
||||
return b
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch, lang: str) -> dict | None:
|
||||
"""One pin as a unified card with its resolved `next`. None when the pinned row has gone (the FK
|
||||
cascade removes the pin itself, so this is only the window between a delete and a refetch)."""
|
||||
card: dict | None = None
|
||||
@@ -115,11 +184,11 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
nxt = dict(card) # a video plays itself
|
||||
|
||||
elif pin.channel_id is not None:
|
||||
ch = db.get(Channel, pin.channel_id)
|
||||
ch = batch.channels.get(pin.channel_id)
|
||||
if ch is None:
|
||||
return None
|
||||
card = cards.from_channel(ch, queries.channel_unwatched_count(db, user, ch.id))
|
||||
row = queries.channel_next_video(db, user, ch.id)
|
||||
count, row = batch.channel_summary.get(ch.id, (0, None))
|
||||
card = cards.from_channel(ch, count)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
elif pin.plex_item_id is not None:
|
||||
@@ -127,7 +196,7 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
if it is None:
|
||||
return None
|
||||
st = rollup.item_state_map(db, user.id, [it.id]).get(it.id)
|
||||
card = cards.from_plex(leaf_card(it, st))
|
||||
card = cards.from_plex(leaf_card(it, st, lang=lang))
|
||||
nxt = dict(card) # a movie or a single episode plays itself
|
||||
|
||||
elif pin.plex_show_id is not None:
|
||||
@@ -135,15 +204,15 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
if sh is None:
|
||||
return None
|
||||
status = rollup.state_map(db, user.id, [sh.id]).get(sh.id, "new")
|
||||
card = cards.from_plex(show_card(sh, status))
|
||||
ep = queries.next_episode(db, user.id, sh.id)
|
||||
card = cards.from_plex(show_card(sh, status, lang=lang))
|
||||
ep = queries.next_episode(db, user.id, sh.id, lang=lang)
|
||||
nxt = cards.from_plex(ep) if ep else None
|
||||
|
||||
elif pin.playlist_id is not None:
|
||||
pl = db.get(Playlist, pin.playlist_id)
|
||||
pl = batch.playlists.get(pin.playlist_id)
|
||||
if pl is None:
|
||||
return None
|
||||
count, row = queries.playlist_summary(db, user, pl.id)
|
||||
count, row = batch.playlist_summary.get(pl.id, (0, None))
|
||||
card = cards.from_playlist(pl, count, row.get("thumbnail_url") if row else None)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
@@ -157,11 +226,24 @@ def _shelf_card(db: Session, user: User, pin: ComfortPin) -> dict | None:
|
||||
return card
|
||||
|
||||
|
||||
def shelf(db: Session, user: User) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order."""
|
||||
_PLEX_COLUMNS = ("plex_item_id", "plex_show_id")
|
||||
|
||||
|
||||
def shelf(db: Session, user: User, *, include_plex: bool = True, lang: str) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order.
|
||||
|
||||
`include_plex=False` is the Home surface with the deployment's Plex toggle off (the owner's call,
|
||||
2026-09-19): the rails beside it go YouTube-only, and a Plex tile there would offer a Play that
|
||||
lands on a gated module. The pins themselves are kept — the pin menus still read the full shelf
|
||||
(`GET /pins`), and switching Plex back on brings the tiles back in their old slots
|
||||
(`reorder` keeps a hidden pin's place)."""
|
||||
out = []
|
||||
for pin in list_pins(db, user):
|
||||
card = _shelf_card(db, user, pin)
|
||||
pins = list_pins(db, user)
|
||||
if not include_plex:
|
||||
pins = [p for p in pins if all(getattr(p, c) is None for c in _PLEX_COLUMNS)]
|
||||
batch = _Batch.load(db, user, pins)
|
||||
for pin in pins:
|
||||
card = _shelf_card(db, user, pin, batch, lang)
|
||||
if card is not None:
|
||||
out.append(card)
|
||||
return out
|
||||
@@ -169,5 +251,17 @@ def shelf(db: Session, user: User) -> list[dict]:
|
||||
|
||||
def playables(shelf_cards: list[dict]) -> list[dict]:
|
||||
"""The `next` of every shelf entry that has one — what the hero may draw from. A pinned channel
|
||||
with nothing unwatched left, or an empty playlist, simply is not an answer right now."""
|
||||
return [c["next"] for c in shelf_cards if c.get("next")]
|
||||
with nothing unwatched left, or an empty playlist, simply is not an answer right now.
|
||||
|
||||
Each carries `pinned_from`: the pin it was resolved from. A show's next EPISODE is not itself
|
||||
pinned, so without it the hero's menu looked the episode up, missed, and offered "Pin" under a
|
||||
kicker that says "From your shelf" — and taking it inserted a second pin. With it, the hero's
|
||||
pin row acts on the shelf entry the card came from (the owner's call, 2026-09-19: "Unpin")."""
|
||||
return [
|
||||
{
|
||||
**c["next"],
|
||||
"pinned_from": {k: c[k] for k in ("source", "kind", "id", "title")},
|
||||
}
|
||||
for c in shelf_cards
|
||||
if c.get("next")
|
||||
]
|
||||
|
||||
@@ -40,11 +40,12 @@ from app.models import (
|
||||
VideoState,
|
||||
)
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, movie_card, show_card
|
||||
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||
from app.plex.query import apply_show_state_filter, content_libs
|
||||
|
||||
# How many rows a single rail may carry. Matches `routes/plex/discover.py`'s RAIL_LIMIT on purpose —
|
||||
# the two surfaces should feel like the same app, and a rail is a rail.
|
||||
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
|
||||
# (`routes/plex/discover.py` imports it): the two surfaces should feel like the same app, and a rail
|
||||
# is a rail. One definition, because a comment was all that held two copies in step.
|
||||
RAIL_LIMIT = 20
|
||||
|
||||
# The feed parameters a Home rail runs with. Spelled out rather than borrowed from the route's
|
||||
@@ -129,18 +130,6 @@ def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) ->
|
||||
return [r for r in out if not r["position_seconds"]][:limit]
|
||||
|
||||
|
||||
def channel_next_video(db: Session, user: User, channel_id: str) -> dict | None:
|
||||
"""The newest unwatched video of a pinned channel — what "put on that channel" resolves to."""
|
||||
# `library_source="all"`: a pinned channel means the whole channel, including videos that only
|
||||
# ever arrived through a search. The feed's "organic" default is a feed-scoping choice, not a
|
||||
# statement about what belongs to a channel.
|
||||
params = dict(_FEED_BASE, channel_ids=[channel_id], scope="all", library_source="all")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys("newest", 0, rank_expr)
|
||||
row = db.execute(query.order_by(*order_by(keys)).limit(1)).first()
|
||||
return serialize(row) if row is not None else None
|
||||
|
||||
|
||||
def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||
"""One video as a feed row, with this user's watch state.
|
||||
|
||||
@@ -158,31 +147,55 @@ def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||
return serialize(row) if row is not None else None
|
||||
|
||||
|
||||
def channel_unwatched_count(db: Session, user: User, channel_id: str) -> int:
|
||||
"""How many unwatched videos a pinned channel is holding — the tile's only honest promise, since a
|
||||
channel has no runtime of its own."""
|
||||
params = dict(_FEED_BASE, channel_ids=[channel_id], scope="all", library_source="all")
|
||||
query, _ = filtered_query(db, user, **params)
|
||||
return int(db.execute(select(func.count()).select_from(query.subquery())).scalar() or 0)
|
||||
def channels_summary(
|
||||
db: Session, user: User, channel_ids: list[str]
|
||||
) -> dict[str, tuple[int, dict | None]]:
|
||||
"""For each pinned channel: (unwatched count, its newest unwatched video) — the tile's only honest
|
||||
promise, since a channel has no runtime of its own, and what "put on that channel" resolves to.
|
||||
|
||||
ONE query for the whole shelf, not two per channel: the count and the first row per channel are
|
||||
window functions over the same filtered set. `library_source="all"`: a pinned channel means the
|
||||
whole channel, including videos that only ever arrived through a search — the feed's "organic"
|
||||
default is a feed-scoping choice, not a statement about what belongs to a channel."""
|
||||
if not channel_ids:
|
||||
return {}
|
||||
params = dict(_FEED_BASE, channel_ids=list(channel_ids), scope="all", library_source="all")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
newest = order_by(sort_keys("newest", 0, rank_expr))
|
||||
per_channel = {"partition_by": Video.channel_id}
|
||||
sub = query.add_columns(
|
||||
func.row_number().over(**per_channel, order_by=newest).label("_rn"),
|
||||
func.count().over(**per_channel).label("_total"),
|
||||
).subquery()
|
||||
rows = db.execute(select(sub).where(sub.c._rn == 1)).all()
|
||||
return {r.channel_id: (int(r._total), serialize(r)) for r in rows}
|
||||
|
||||
|
||||
def playlist_summary(db: Session, user: User, playlist_id: int) -> tuple[int, dict | None]:
|
||||
"""(item count, the row to play next) for a pinned playlist — its first not-yet-watched item, or
|
||||
its first item once the whole list is watched. The same row doubles as the tile's artwork, so a
|
||||
shelf of playlists is scannable instead of a row of blank boxes."""
|
||||
def playlists_summary(
|
||||
db: Session, user: User, playlist_ids: list[int]
|
||||
) -> dict[int, tuple[int, dict | None]]:
|
||||
"""For each pinned playlist: (item count, the row to play next) — its first not-yet-watched item,
|
||||
or its first item once the whole list is watched. The same row doubles as the tile's artwork, so
|
||||
a shelf of playlists is scannable instead of a row of blank boxes. One query for the whole shelf."""
|
||||
if not playlist_ids:
|
||||
return {}
|
||||
st = aliased(VideoState)
|
||||
rows = db.execute(
|
||||
select(*feed_columns(user, st), PlaylistItem.position)
|
||||
select(*feed_columns(user, st), PlaylistItem.playlist_id.label("_playlist"))
|
||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(st, and_(st.video_id == Video.id, st.user_id == user.id))
|
||||
.where(PlaylistItem.playlist_id == playlist_id)
|
||||
.order_by(PlaylistItem.position.asc())
|
||||
.where(PlaylistItem.playlist_id.in_(playlist_ids))
|
||||
.order_by(PlaylistItem.playlist_id, PlaylistItem.position.asc())
|
||||
).all()
|
||||
if not rows:
|
||||
return 0, None
|
||||
nxt = next((r for r in rows if (r.status or "new") not in ("watched", "hidden")), rows[0])
|
||||
return len(rows), serialize(nxt)
|
||||
grouped: dict[int, list] = {}
|
||||
for r in rows:
|
||||
grouped.setdefault(r._playlist, []).append(r)
|
||||
out: dict[int, tuple[int, dict | None]] = {}
|
||||
for pid, items in grouped.items():
|
||||
nxt = next((r for r in items if (r.status or "new") not in ("watched", "hidden")), items[0])
|
||||
out[pid] = (len(items), serialize(nxt))
|
||||
return out
|
||||
|
||||
|
||||
# ── Plex ──────────────────────────────────────────────────────────────────────────────────────────
|
||||
@@ -208,7 +221,9 @@ def _unseen_movie_conds(uid: int):
|
||||
return ~seen
|
||||
|
||||
|
||||
def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tuple[object, dict]]:
|
||||
def plex_in_progress(
|
||||
db: Session, uid: int, limit: int = RAIL_LIMIT, *, lang: str
|
||||
) -> list[tuple[object, dict]]:
|
||||
"""In-progress movies + partially-watched shows as (progressed_at, card).
|
||||
|
||||
Mirrors `routes/plex/discover.py::_continue_watching` — same `PlexState` predicate for movies (a
|
||||
@@ -236,7 +251,7 @@ def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tup
|
||||
.limit(limit)
|
||||
)
|
||||
for it, st in db.execute(mq).all():
|
||||
out.append((st.progress_updated_at, movie_card(it, st)))
|
||||
out.append((st.progress_updated_at, movie_card(it, st, lang=lang)))
|
||||
|
||||
if show_libs:
|
||||
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
||||
@@ -250,12 +265,14 @@ def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tup
|
||||
for sh, resumed_at in db.execute(
|
||||
sq.order_by(prog.c.resumed_at.desc().nullslast()).limit(limit)
|
||||
).all():
|
||||
out.append((resumed_at, show_card(sh, "in_progress")))
|
||||
out.append((resumed_at, show_card(sh, "in_progress", lang=lang)))
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) -> list[dict]:
|
||||
def plex_surprise(
|
||||
db: Session, uid: int, budget: str, seed: int, limit: int, *, lang: str
|
||||
) -> list[dict]:
|
||||
"""Unseen playable movies inside the budget, in seeded-shuffle order.
|
||||
|
||||
Movies only. A show is not a thing you can start playing, and resolving every candidate show to
|
||||
@@ -287,14 +304,14 @@ def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) ->
|
||||
q = q.where(PlexItem.duration_s <= hi)
|
||||
items = db.execute(q).scalars().all()
|
||||
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||
return [movie_card(it, states.get(it.id)) for it in items]
|
||||
return [movie_card(it, states.get(it.id), lang=lang) for it in items]
|
||||
|
||||
|
||||
# Watch states that take an episode out of the running for "what would play next".
|
||||
_DONE = ("watched", "hidden")
|
||||
|
||||
|
||||
def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
||||
def next_episode(db: Session, uid: int, show_id: int, *, lang: str) -> dict | None:
|
||||
"""What "put that show on" means: the episode in progress, else the first unwatched one, else the
|
||||
first — over the same `(season, episode)` broadcast ordering `detail.py::_episode_nav` uses.
|
||||
|
||||
@@ -344,5 +361,9 @@ def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
||||
ep, st = pick
|
||||
show = db.get(PlexShow, show_id)
|
||||
return leaf_card(
|
||||
ep, st, show_title=show.title if show else None, show_rk=show.rating_key if show else None
|
||||
ep,
|
||||
st,
|
||||
show_title=display_title(show, lang) if show else None,
|
||||
show_rk=show.rating_key if show else None,
|
||||
lang=lang,
|
||||
)
|
||||
|
||||
@@ -22,7 +22,7 @@ def plex_available(db: Session) -> bool:
|
||||
return sysconfig.get_bool(db, "plex_enabled")
|
||||
|
||||
|
||||
def playable(db: Session, uid: int, card: dict) -> dict | None:
|
||||
def playable(db: Session, uid: int, card: dict, *, lang: str) -> dict | None:
|
||||
"""The thing that would actually START from this card — itself for anything with a runtime, and
|
||||
for a Plex SHOW the episode that is next (in progress, else the first unwatched, else the first).
|
||||
|
||||
@@ -40,11 +40,11 @@ def playable(db: Session, uid: int, card: dict) -> dict | None:
|
||||
sh = by_rating_key(db, PlexShow, card["id"])
|
||||
if sh is None:
|
||||
return None
|
||||
ep = queries.next_episode(db, uid, sh.id)
|
||||
ep = queries.next_episode(db, uid, sh.id, lang=lang)
|
||||
return cards.from_plex(ep) if ep else None
|
||||
|
||||
|
||||
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[dict]:
|
||||
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT, *, lang: str) -> list[dict]:
|
||||
"""One resume rail across both sources, most recently progressed first.
|
||||
|
||||
A Plex SHOW enters as a show CARD — that is what the Plex rail shows, and it is the artwork the
|
||||
@@ -57,17 +57,22 @@ def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[
|
||||
(ts, cards.from_video(r)) for ts, r in queries.yt_in_progress(db, user, limit)
|
||||
]
|
||||
if plex_available(db):
|
||||
rows += [(ts, cards.from_plex(c)) for ts, c in queries.plex_in_progress(db, user.id, limit)]
|
||||
rows += [
|
||||
(ts, cards.from_plex(c))
|
||||
for ts, c in queries.plex_in_progress(db, user.id, limit, lang=lang)
|
||||
]
|
||||
rows.sort(key=lambda r: r[0] or NO_TS, reverse=True)
|
||||
out = [card for _, card in rows[:limit]]
|
||||
# Resolve AFTER the slice: only the rows that survive it are worth a query.
|
||||
for card in out:
|
||||
if card["kind"] == "show":
|
||||
card["next"] = playable(db, user.id, card)
|
||||
card["next"] = playable(db, user.id, card, lang=lang)
|
||||
return out
|
||||
|
||||
|
||||
def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) -> list[dict]:
|
||||
def surprise(
|
||||
db: Session, user: User, budget: str, seed: int, limit: int = 3, *, lang: str
|
||||
) -> list[dict]:
|
||||
"""The hero's draw pool: unseen things that fit the budget, from both sources under ONE seed.
|
||||
|
||||
Interleaved rather than concatenated — the same alternation `routes/plex/discover.py` uses for its
|
||||
@@ -83,7 +88,9 @@ def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) ->
|
||||
(cards.from_video, queries.yt_surprise(db, user, budget, seed, limit)),
|
||||
(
|
||||
cards.from_plex,
|
||||
queries.plex_surprise(db, user.id, budget, seed, limit) if plex_available(db) else [],
|
||||
queries.plex_surprise(db, user.id, budget, seed, limit, lang=lang)
|
||||
if plex_available(db)
|
||||
else [],
|
||||
),
|
||||
]
|
||||
if seed % 2:
|
||||
|
||||
@@ -11,9 +11,22 @@ Resolution works both before extraction (from source_kind / URL host) and after
|
||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||
"""
|
||||
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads.stated import StatedFailure
|
||||
|
||||
# Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry
|
||||
# below and the failure catalog (errors.py) decide on it, so the wording exists once.
|
||||
FACEBOOK_PARSE_WALL = "cannot parse data"
|
||||
# A source's own verdicts on a link, raised from `resolve_url`. errors.py classifies them; it imports
|
||||
# them from here because this module must not import errors (errors → credentials → adapters).
|
||||
POST_WITHOUT_VIDEO_MESSAGE = "No single video was found in this post."
|
||||
POST_NOT_SHOWN_MESSAGE = "Facebook did not show this post to the signed-in session."
|
||||
|
||||
|
||||
class SourceAdapter:
|
||||
@@ -22,9 +35,21 @@ class SourceAdapter:
|
||||
key = "generic"
|
||||
|
||||
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||||
def client_sets(self) -> list:
|
||||
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
|
||||
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
"""The link to hand yt-dlp for `url`, resolved BEFORE the first attempt. `ydl_opts` are the
|
||||
download's own options (cookie, timeouts, address family) for any page this needs to read.
|
||||
|
||||
Raises a RuntimeError carrying one of this module's verdicts when the page read shows the
|
||||
download cannot go ahead: the content is there but holds no single video
|
||||
(`POST_WITHOUT_VIDEO_MESSAGE`), or the page is not the content at all — the login page a
|
||||
refused cookie gets (`POST_NOT_SHOWN_MESSAGE`), never the no-video verdict. A failed page read
|
||||
is raised as itself, so it is classified like any other download error. Generic: `url`."""
|
||||
return url
|
||||
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
"""Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one
|
||||
is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`).
|
||||
`authed` is whether the acting user's cookie file goes with the download."""
|
||||
return [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
@@ -37,6 +62,10 @@ class SourceAdapter:
|
||||
return False
|
||||
|
||||
# --- metadata -----------------------------------------------------------------------------
|
||||
def title(self, info: dict) -> str | None:
|
||||
"""The download's raw title, before `titles.normalize_title` cleans it. Generic: yt-dlp's."""
|
||||
return info.get("title")
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||||
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||||
@@ -65,12 +94,46 @@ class YouTubeAdapter(SourceAdapter):
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
# Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then
|
||||
# refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves
|
||||
# between the player response and the stream, so this is the same per-client flakiness one
|
||||
# step later — and it used to end the job on attempt 1, because the text classifies as
|
||||
# `network` and nothing here matched it. The format URLs belong to the client that minted
|
||||
# them, so the next client set is a real second chance.
|
||||
"unable to download video data",
|
||||
)
|
||||
# Measured in the worker against a REAL account cookie (2026-09-16, yt-dlp 2026.07.04): with
|
||||
# `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
|
||||
# cookies`, drops it, and the request that is left with no client fails with "Requested format is
|
||||
# not available" — which is already a retriable marker above, so the cost is a wasted attempt
|
||||
# rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every
|
||||
# signed-in download would burn its first attempt that way before reaching the POT-backed web
|
||||
# fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
|
||||
# ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
|
||||
# that moves the table turns the gate red instead of silently un-authing every YouTube download.
|
||||
_COOKIE_CAPABLE = frozenset(
|
||||
{
|
||||
"mweb",
|
||||
"tv",
|
||||
"tv_downgraded",
|
||||
"web",
|
||||
"web_creator",
|
||||
"web_embedded",
|
||||
"web_music",
|
||||
"web_safari",
|
||||
}
|
||||
)
|
||||
# Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all.
|
||||
# Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for
|
||||
# the per-client flakiness that the two-tier ladder gives a signed-out download.
|
||||
_AUTHED_DEFAULT = ("web_safari", "web")
|
||||
|
||||
def client_sets(self) -> list:
|
||||
return [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c]
|
||||
if authed:
|
||||
kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets]
|
||||
return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)]
|
||||
return sets or [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||||
@@ -93,6 +156,63 @@ class YouTubeAdapter(SourceAdapter):
|
||||
|
||||
class FacebookAdapter(SourceAdapter):
|
||||
key = "facebook"
|
||||
# Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||||
# with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a
|
||||
# SECOND attempt behind that one error, not the default: the links that already download with the
|
||||
# cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without
|
||||
# a cookie the same error is the login wall, which no disguise gets past.
|
||||
_IMPERSONATE = "impersonate"
|
||||
# A post page is ~4–5 MB of inline JSON; anything far past that is not a post page.
|
||||
_POST_PAGE_MAX_BYTES = 16 * 1024 * 1024
|
||||
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
# Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
|
||||
# downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
|
||||
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
|
||||
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
|
||||
post_id = facebook_group_post_id(url)
|
||||
if post_id is None or not ydl_opts.get("cookiefile"):
|
||||
return url
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
|
||||
page = raw.decode("utf-8", "replace")
|
||||
# Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
|
||||
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
|
||||
if _quoted(post_id) not in page:
|
||||
raise StatedFailure(POST_NOT_SHOWN_MESSAGE)
|
||||
video_id = facebook_post_video_id(page, post_id)
|
||||
if video_id is None:
|
||||
# Never fall back to the post link: that is exactly where the wrong video comes from.
|
||||
raise StatedFailure(POST_WITHOUT_VIDEO_MESSAGE)
|
||||
return f"https://www.facebook.com/watch/?v={video_id}"
|
||||
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
return [None, self._IMPERSONATE] if authed else [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
if clients == self._IMPERSONATE:
|
||||
return {"impersonate": ImpersonateTarget.from_str("chrome")}
|
||||
return {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
return FACEBOOK_PARSE_WALL in error_text.lower()
|
||||
|
||||
# Measured (2026-09-14): a SIGNED-IN read gives the placeholder "Video" (plain and as Chrome), and an
|
||||
# 18+ reel only readable as Chrome gives "Facebook"; the post's text is still in `description`. A
|
||||
# signed-out read titles the page with that same text behind the engagement counts, cut short.
|
||||
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
|
||||
|
||||
def title(self, info: dict) -> str | None:
|
||||
raw = info.get("title")
|
||||
if (raw or "").strip().lower() not in self._PLACEHOLDER_TITLES:
|
||||
return raw
|
||||
line = next(
|
||||
(ln.strip() for ln in (info.get("description") or "").splitlines() if ln.strip()), ""
|
||||
)
|
||||
# The whole first line, never cut here with an ellipsis: normalize_title's NFKC turns "…" into
|
||||
# "..." and then collapses it to ".", which read as a finished sentence ("… live at."). The
|
||||
# stored title is unbounded; display_name and the file name clip it themselves.
|
||||
return line or raw # no description either (the 18+ reel): nothing better on the page
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||||
@@ -107,6 +227,39 @@ class FacebookAdapter(SourceAdapter):
|
||||
return True
|
||||
|
||||
|
||||
_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
|
||||
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
|
||||
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')
|
||||
|
||||
|
||||
def _quoted(post_id: str) -> str:
|
||||
# How the page JSON carries the id: quoted, so a longer id that merely starts with it never counts.
|
||||
return f'"{post_id}"'
|
||||
|
||||
|
||||
def facebook_group_post_id(url: str) -> str | None:
|
||||
"""The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None."""
|
||||
parsed = urlparse(url)
|
||||
if not _host_is(parsed.netloc.lower(), "facebook.com"):
|
||||
return None
|
||||
match = _FB_GROUP_POST_PATH.match(parsed.path)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def facebook_post_video_id(page: str, post_id: str) -> str | None:
|
||||
"""The id of the one video attached to `post_id`, read from its page: the Video nodes of the script
|
||||
blocks that carry the post id (the page's other ~10 videos — feed, reels — sit in other blocks).
|
||||
None for no video AND for more than one: guessing between two would be the wrong-video bug again."""
|
||||
quoted = _quoted(post_id)
|
||||
ids = {
|
||||
vid
|
||||
for block in _SCRIPT_BLOCK.findall(page)
|
||||
if quoted in block
|
||||
for vid in _VIDEO_NODE.findall(block)
|
||||
}
|
||||
return ids.pop() if len(ids) == 1 else None
|
||||
|
||||
|
||||
class RedditAdapter(SourceAdapter):
|
||||
key = "reddit"
|
||||
|
||||
|
||||
@@ -11,11 +11,12 @@ Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
@@ -23,12 +24,17 @@ from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
||||
needs the POT/player-client strategy even just to extract)."""
|
||||
with formats.hiding(url):
|
||||
return _extract_meta_scoped(url, adapter, asset_id)
|
||||
|
||||
|
||||
def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
||||
for clients in adapter.client_sets():
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
||||
@@ -40,7 +46,19 @@ def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
if info:
|
||||
return info
|
||||
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
|
||||
log.info("meta extract failed for %s (%s): %s", url, clients, str(exc)[:100])
|
||||
# The asset id, not the URL. A source URL is a row of this user's download
|
||||
# history, and `redact_text` only takes credential params out of it — the rest
|
||||
# of it goes in whole. The id names the same asset to anyone holding the
|
||||
# database and nothing at all to anyone who is not; the host is kept because it
|
||||
# is what decides which client set was tried. yt-dlp's message goes through
|
||||
# `log_safe` for the same reason: it opens with the URL or the video id.
|
||||
log.info(
|
||||
"meta extract failed for asset %s on %s (%s): %s",
|
||||
asset_id,
|
||||
urlsplit(url).hostname or "?", # log-pii-ok: the host, never the path or query
|
||||
clients,
|
||||
formats.log_safe(exc)[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -60,12 +78,14 @@ def run() -> dict:
|
||||
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
||||
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||
info = _extract_meta(url, adapter)
|
||||
info = _extract_meta(url, adapter, a.id)
|
||||
uu = adapter.uploader_url(info) if info else None
|
||||
if uu:
|
||||
a.uploader_url = uu
|
||||
filled_url += 1
|
||||
log.info("asset %s uploader_url <- %s", a.id, uu)
|
||||
log.info(
|
||||
"asset %s uploader_url filled", a.id
|
||||
) # the URL names what they downloaded
|
||||
if a.poster_path is None:
|
||||
# Record a self-hosted poster for every download (the reliable og:image source):
|
||||
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
||||
@@ -73,7 +93,7 @@ def run() -> dict:
|
||||
if rel:
|
||||
a.poster_path = rel
|
||||
filled_poster += 1
|
||||
log.info("asset %s poster <- %s", a.id, rel)
|
||||
log.info("asset %s poster filled", a.id) # the path carries the title
|
||||
db.commit()
|
||||
result = {
|
||||
"assets": len(assets),
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
"""Per-user, per-host download cookies — the single source of truth for "does the acting user's
|
||||
cookie apply to this download, and how does that keep the result private".
|
||||
|
||||
Two call sites depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
Two sides depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
in prose):
|
||||
* `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user.
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user. Retry
|
||||
(`service.resume`) and Re-download (`service.redownload`) ask the same question again, so a
|
||||
cookie the user added after a failure goes with the next attempt.
|
||||
* the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it
|
||||
is downloading into is that private, salted identity — so a cookie can never poison a shared
|
||||
asset (the load-bearing cross-user privacy fix).
|
||||
|
||||
Both go through `cookie_salt` / `authed_format_sig` / `host_key_for_url` here, so they can't drift.
|
||||
Both key the private identity on `cookie_salt` / `host_key_for_url` here, so they can't drift.
|
||||
|
||||
Security controls enforced here:
|
||||
* **host scope (SEC-12)** — `host_key_for_url` matches the download URL's netloc against a curated
|
||||
@@ -21,19 +23,19 @@ Security controls enforced here:
|
||||
logged, or written to a job's error/snapshot.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential
|
||||
from app.models import DownloadCredential, MediaAsset
|
||||
from app.security import decrypt_optional, encrypt
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.credentials")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class HostSpec:
|
||||
@@ -53,6 +55,11 @@ class HostSpec:
|
||||
# all derive from this, so there is nothing else to wire up.
|
||||
SUPPORTED_HOSTS: tuple[HostSpec, ...] = (
|
||||
HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")),
|
||||
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
||||
# job 273: every player client fails on that one video while its neighbours download fine, and
|
||||
# the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a
|
||||
# row here the `login_required` copy sent the user to a dropdown that could not help them.
|
||||
HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")),
|
||||
HostSpec("instagram", "Instagram", ("instagram.com",)),
|
||||
HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")),
|
||||
HostSpec("tiktok", "TikTok", ("tiktok.com",)),
|
||||
@@ -99,11 +106,38 @@ def cookie_salt(user_id: int, host_key: str) -> str:
|
||||
|
||||
|
||||
def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str:
|
||||
"""The salted `format_sig` an authed download for (user, host) resolves to — the private asset
|
||||
identity. Both enqueue (to create it) and the worker (to recognise it) compute it HERE."""
|
||||
"""The salted `format_sig` an authed download for (user, host) resolves to under the current
|
||||
signature version — the private asset enqueue creates, and one of the identities `is_authed_asset`
|
||||
recognises. Neither calls it; the tests use it to hold the two to each other."""
|
||||
return formats.format_sig(spec, salt=cookie_salt(user_id, host_key))
|
||||
|
||||
|
||||
def rebuild_salt(asset_id: int, cookie: str | None) -> str:
|
||||
"""The salt of a Re-download's fresh asset: its OWN row id (so every rebuild is a new identity no
|
||||
enqueue can ever compute), prefixed with the user's `cookie_salt` when a cookie applies. The id is
|
||||
what makes it recognisable — the worker has the asset row, so `is_authed_asset` can recompute this
|
||||
one, where a salt built from the job's previous asset id could not be."""
|
||||
return f"{cookie}:rebuild:{asset_id}" if cookie else f"rebuild:{asset_id}"
|
||||
|
||||
|
||||
def is_authed_asset(asset: MediaAsset, spec: dict, user_id: int, url: str | None) -> bool:
|
||||
"""Whether an asset is `user_id`'s private, cookie-salted identity for `url`'s host — the one
|
||||
kind of asset a cookie is ever attached to: the one enqueue creates (`authed_format_sig`), or a
|
||||
Re-download's rebuild of it (`rebuild_salt`). The worker's cookie gate asks it before a download,
|
||||
and the failure diagnosis asks it after one (a failure on this asset happened WITH the cookie).
|
||||
Either signature version counts (`formats.format_sigs`): a job queued before a deploy that bumped
|
||||
the version still downloads into the asset it was given."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return False
|
||||
salt = cookie_salt(user_id, host_key)
|
||||
if asset.format_sig in formats.format_sigs(spec, salt=salt):
|
||||
return True
|
||||
return asset.id is not None and asset.format_sig in formats.format_sigs(
|
||||
spec, salt=rebuild_salt(asset.id, salt)
|
||||
)
|
||||
|
||||
|
||||
def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
||||
"""At enqueue: the cache salt to apply if this user has a cookie for `url`'s host, else None (a
|
||||
normal shared download). Returns the salt string so the caller passes it straight to
|
||||
@@ -199,20 +233,19 @@ def status_list(db: Session, user_id: int) -> list[dict]:
|
||||
# --- worker: resolve + write the cookie file --------------------------------------------------
|
||||
|
||||
|
||||
def resolve_cookiefile(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
url: str | None,
|
||||
asset_format_sig: str,
|
||||
spec: dict,
|
||||
staging: Path,
|
||||
) -> Path | None:
|
||||
"""In the worker child: the cookies.txt to hand yt-dlp for this download, or None.
|
||||
def usable_cookie(
|
||||
db: Session, user_id: int, url: str | None, asset: MediaAsset, spec: dict
|
||||
) -> str | None:
|
||||
"""The user's cookie text a download of `url` into `asset` goes with, or None — the ONE decision
|
||||
whether a download is signed in. The worker child writes it out for yt-dlp (`resolve_cookiefile`);
|
||||
the worker asks it again for a job that inherits a failed asset. That second ask sees the
|
||||
credential as it is when the job is claimed: a cookie the user removed or added since the failed
|
||||
download changes this job's explanation to fit the cookie its Retry would go with.
|
||||
|
||||
The double gate that prevents cross-user leakage:
|
||||
1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND
|
||||
2. the asset being downloaded into must BE this user's private, salted identity
|
||||
(`asset_format_sig == authed_format_sig(...)`).
|
||||
(`is_authed_asset`).
|
||||
|
||||
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
||||
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
||||
@@ -224,18 +257,31 @@ def resolve_cookiefile(
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
if cred is None:
|
||||
return None
|
||||
if asset_format_sig != authed_format_sig(spec, user_id, host_key):
|
||||
if not is_authed_asset(asset, spec, user_id, url):
|
||||
return None # shared asset — never attach a cookie to a non-private identity
|
||||
raw = decrypt_optional(cred.cookies_enc)
|
||||
if not raw:
|
||||
log.warning(
|
||||
"download cookie for user %s host %s could not be decrypted (rotated key?) — "
|
||||
"proceeding without it",
|
||||
"treated as no cookie",
|
||||
user_id,
|
||||
host_key,
|
||||
)
|
||||
return None
|
||||
return _write_cookiefile(raw, staging)
|
||||
return raw
|
||||
|
||||
|
||||
def resolve_cookiefile(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
url: str | None,
|
||||
asset: MediaAsset,
|
||||
spec: dict,
|
||||
staging: Path,
|
||||
) -> Path | None:
|
||||
"""In the worker child: the cookies.txt to hand yt-dlp for this download (`usable_cookie`), or None."""
|
||||
raw = usable_cookie(db, user_id, url, asset, spec)
|
||||
return _write_cookiefile(raw, staging) if raw else None
|
||||
|
||||
|
||||
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
||||
|
||||
@@ -23,7 +23,6 @@ for a player hover-preview later.
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
@@ -33,9 +32,11 @@ import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.downloads.stated import StatedFailure
|
||||
|
||||
log = logging.getLogger("siftlode.edit")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def encode_thread_cap() -> int:
|
||||
@@ -71,7 +72,13 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
if crop:
|
||||
try:
|
||||
c = {k: int(round(float(crop[k]))) for k in ("x", "y", "w", "h")}
|
||||
except (KeyError, TypeError, ValueError):
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
# The clip still renders — uncropped. The user asked for a crop, got a whole frame,
|
||||
# and nothing anywhere called it a failure; this line is the only trace there is.
|
||||
log.warning(
|
||||
"the crop in an edit spec could not be read and was dropped",
|
||||
extra=logs.event("edit.spec.crop_dropped", reason=type(exc).__name__),
|
||||
)
|
||||
c = None
|
||||
if c and c["w"] > 0 and c["h"] > 0 and c["x"] >= 0 and c["y"] >= 0:
|
||||
out["crop"] = c
|
||||
@@ -80,6 +87,7 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
segs = spec.get("segments")
|
||||
if isinstance(segs, list) and segs:
|
||||
norm: list[dict] = []
|
||||
unreadable = 0
|
||||
for s in segs:
|
||||
try:
|
||||
st = max(0.0, float((s or {}).get("start_s") or 0.0))
|
||||
@@ -88,9 +96,18 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
continue
|
||||
en = float(end_raw)
|
||||
except (TypeError, ValueError):
|
||||
# silent-ok: counted, not logged here — one cut-list can drop several, and the
|
||||
# join that comes out is simply shorter than the one the user built. Reported
|
||||
# once after the loop.
|
||||
unreadable += 1
|
||||
continue
|
||||
if en > st:
|
||||
norm.append({"start_s": round(st, 3), "end_s": round(en, 3)})
|
||||
if unreadable:
|
||||
log.warning(
|
||||
"some segments of a cut list could not be read and were dropped",
|
||||
extra=logs.event("edit.spec.segments_dropped", dropped=unreadable, kept=len(norm)),
|
||||
)
|
||||
norm.sort(key=lambda x: x["start_s"])
|
||||
# SEC-13: an accurate join builds one ffmpeg `filter_complex` with two filters per segment,
|
||||
# so an unbounded list becomes an unbounded command line and an ffmpeg that never finishes.
|
||||
@@ -114,7 +131,13 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
start = max(0.0, float(trim.get("start_s") or 0.0))
|
||||
end_raw = trim.get("end_s")
|
||||
end = float(end_raw) if end_raw is not None else None
|
||||
except (TypeError, ValueError):
|
||||
except (TypeError, ValueError) as exc:
|
||||
# Either a 400 the user cannot act on, or — beside a crop — a full-length clip where
|
||||
# they asked for a cut. Both are worth one line saying the spec was the problem.
|
||||
log.warning(
|
||||
"the trim in an edit spec could not be read and was dropped",
|
||||
extra=logs.event("edit.spec.trim_dropped", reason=type(exc).__name__),
|
||||
)
|
||||
start, end = 0.0, None
|
||||
t: dict = {"start_s": round(start, 3)}
|
||||
if end is not None and end > start:
|
||||
@@ -306,11 +329,14 @@ def _parse_out_time(line: str) -> float | None:
|
||||
if line.startswith("out_time_us="):
|
||||
try:
|
||||
return int(line.split("=", 1)[1]) / 1_000_000
|
||||
# silent-ok: one line of ffmpeg's `-progress` stream, several per second. None means "this
|
||||
# line carried no usable time", which the caller already treats as "no progress yet".
|
||||
except ValueError:
|
||||
return None
|
||||
if line.startswith("out_time_ms="): # (some builds mislabel this as microseconds)
|
||||
try:
|
||||
return int(line.split("=", 1)[1]) / 1_000_000
|
||||
# silent-ok: as above, the other spelling of the same line.
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
@@ -329,7 +355,9 @@ def _pump(stream, sink: "queue.Queue[str | None]") -> None:
|
||||
try:
|
||||
for line in stream:
|
||||
sink.put(line)
|
||||
except (ValueError, OSError): # pipe closed under us by the kill path
|
||||
# silent-ok: the pipe was closed under us by our OWN kill path — the expected end of a
|
||||
# cancelled or timed-out run, and the code that did the killing is the code that reports it.
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
finally:
|
||||
sink.put(None)
|
||||
@@ -347,6 +375,8 @@ def _drain_tail(stream, sink: list[str]) -> None:
|
||||
sink.append(line)
|
||||
if len(sink) > 200:
|
||||
del sink[:100]
|
||||
# silent-ok: same closed pipe as `_pump` above, and the tail collected so far is still handed
|
||||
# to whoever reports the failure.
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
@@ -362,13 +392,17 @@ def escalate_kill(proc: subprocess.Popen, terminate, kill, grace_s: float, label
|
||||
try:
|
||||
proc.wait(timeout=grace_s)
|
||||
return
|
||||
# silent-ok: the escalation below IS the handling of this timeout, and surviving THAT is what
|
||||
# the line under the second wait reports.
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
kill()
|
||||
try:
|
||||
proc.wait(timeout=grace_s)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid)
|
||||
# noqa TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
||||
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
||||
|
||||
|
||||
def _stop(proc: subprocess.Popen) -> None:
|
||||
@@ -404,6 +438,8 @@ def run_ffmpeg(
|
||||
while True:
|
||||
try:
|
||||
line = lines.get(timeout=1.0)
|
||||
# silent-ok: not a failure — the one-second timeout is how this loop gets a turn to
|
||||
# poll for cancellation while ffmpeg is quiet.
|
||||
except queue.Empty:
|
||||
line = ""
|
||||
if line is None: # stdout hit EOF — ffmpeg is done (or dying)
|
||||
@@ -416,11 +452,11 @@ def run_ffmpeg(
|
||||
if should_cancel():
|
||||
raise EditAborted
|
||||
if time.monotonic() - last_output > stall_timeout_s:
|
||||
raise RuntimeError(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
|
||||
raise StatedFailure(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
|
||||
try:
|
||||
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
|
||||
except subprocess.TimeoutExpired: # pipes closed but the process lingers
|
||||
raise RuntimeError("ffmpeg did not exit after closing its output")
|
||||
raise StatedFailure("ffmpeg did not exit after closing its output")
|
||||
except BaseException:
|
||||
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
|
||||
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
|
||||
@@ -435,7 +471,7 @@ def run_ffmpeg(
|
||||
if pipe:
|
||||
pipe.close()
|
||||
if ret != 0:
|
||||
raise RuntimeError(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
raise StatedFailure(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
|
||||
|
||||
# --- ffmpeg: universal browser playability --------------------------------------------------
|
||||
@@ -471,7 +507,13 @@ def run_ffprobe_json(
|
||||
try:
|
||||
out = subprocess.run(args, capture_output=True, text=True, timeout=30, check=True).stdout
|
||||
return json.loads(out)
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError) as exc:
|
||||
# None makes every field it would have filled read as absent, and a file with no streams
|
||||
# and no duration is indistinguishable here from one that is not media at all.
|
||||
log.warning(
|
||||
"ffprobe could not read a file",
|
||||
extra=logs.event("media.ffprobe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -525,7 +567,13 @@ def probe_duration(path: Path) -> int:
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
return int(float(out)) if out else 0
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError) as exc:
|
||||
# Zero is the denominator of the re-encode progress bar, so a failed probe leaves the user
|
||||
# watching a bar that never moves rather than a job that says it went wrong.
|
||||
log.info(
|
||||
"the duration probe failed; progress for this job has no total",
|
||||
extra=logs.event("media.duration_probe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -670,15 +718,28 @@ def _run_ffmpeg_best_effort(cmd: list[str], timeout_s: int) -> Exception | None:
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, timeout=timeout_s, check=True)
|
||||
return None
|
||||
# silent-ok: the exception is the RETURN VALUE. This helper exists so the three callers share
|
||||
# one run/timeout policy, and each logs the failure with the context only it has.
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError) as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _step_failure(err: BaseException) -> str:
|
||||
"""What a best-effort image step's failure line says: the kind and, for ffmpeg, its exit
|
||||
status. Never `str(err)`: a CalledProcessError or TimeoutExpired quotes ffmpeg's argv, and an
|
||||
OSError its file name — the source path either way, which is built from the channel and the
|
||||
title."""
|
||||
code = getattr(err, "returncode", None)
|
||||
return f"{type(err).__name__} {code}" if code is not None else type(err).__name__
|
||||
|
||||
|
||||
def _is_jpeg(path: Path) -> bool:
|
||||
"""True if the file starts with the JPEG SOI marker (FF D8 FF)."""
|
||||
try:
|
||||
with path.open("rb") as f:
|
||||
return f.read(3) == b"\xff\xd8\xff"
|
||||
# silent-ok: False only means "do not copy this straight through" — the caller then goes the
|
||||
# ffmpeg route, which opens the same file and reports it properly if it really is unreadable.
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
@@ -699,13 +760,15 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
||||
shutil.copyfile(src, dest)
|
||||
return dest
|
||||
except OSError as exc:
|
||||
log.info("thumbnail copy failed (%s); poster falls back to a cut frame", str(exc)[:120])
|
||||
log.info(
|
||||
"thumbnail copy failed (%s); poster falls back to a cut frame", _step_failure(exc)
|
||||
)
|
||||
return None
|
||||
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
|
||||
if err is not None:
|
||||
log.info(
|
||||
"thumbnail->jpg conversion failed (%s); poster falls back to a cut frame",
|
||||
str(err)[:120],
|
||||
_step_failure(err),
|
||||
)
|
||||
return None
|
||||
return dest if dest.exists() else None
|
||||
@@ -730,7 +793,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
at = min(3.0, (asset.duration_s or 10) * 0.1)
|
||||
err = _run_ffmpeg_best_effort(build_poster_cmd(src, dest, at), timeout_s)
|
||||
if err is not None:
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, str(err)[:120])
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, _step_failure(err))
|
||||
return None
|
||||
return rel if dest.exists() else None
|
||||
|
||||
@@ -761,7 +824,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict |
|
||||
cmd = build_storyboard_cmd(src, dest, asset.duration_s, geom)
|
||||
err = _run_ffmpeg_best_effort(cmd, timeout_s)
|
||||
if err is not None:
|
||||
log.info("storyboard gen skipped for asset %s: %s", asset.id, str(err)[:120])
|
||||
log.info("storyboard gen skipped for asset %s: %s", asset.id, _step_failure(err))
|
||||
return None
|
||||
if not dest.exists():
|
||||
return None
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code.
|
||||
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code — and, where a site's
|
||||
failure has been measured, into that site's entry of the failure catalog.
|
||||
|
||||
The worker stores the raw exception text (truncated) for technical detail, but a raw
|
||||
`ERROR: [youtube] <id>: Private video. Sign in…` is noise to a user. `classify()` maps that text
|
||||
@@ -6,21 +7,51 @@ to one of a small set of category codes; the frontend turns the code into a loca
|
||||
explanation (and decides whether a retry makes sense). The raw text stays available behind a
|
||||
"details" disclosure. This is the same string-matching approach the worker already trusts for
|
||||
`_RETRIABLE_MARKERS` — kept here as the single source of truth for "what kind of failure is this".
|
||||
|
||||
**The failure catalog** (`_CATALOG`, E11.5). A category is generic: "Sign-in required" cannot tell a
|
||||
Facebook user that the cookie they already gave was refused, and yt-dlp words several different
|
||||
Facebook walls identically (`Cannot parse data`). A catalog entry is scoped to ONE host and may also
|
||||
depend on whether a cookie was attached and on the kind of link, and names a `detail` code the UI has
|
||||
its own sentence for — why it failed on that site, and what the user can do. `diagnose()` tries the
|
||||
catalog first and falls back to the category. How an entry is added, every time a failure is reported:
|
||||
1. reproduce it on dev (with and without the user's cookie) and keep the verbatim yt-dlp text;
|
||||
2. add the entry below, scoped as narrowly as the measurement allows (a marker that is yt-dlp's
|
||||
generic wording for every site must never match without its host; a marker another module also
|
||||
decides on — the Facebook Chrome retry's FACEBOOK_PARSE_WALL — is imported, never retyped);
|
||||
3. pin the real text in `tests/test_download_errors.py`;
|
||||
4. add `downloads.errors.fail.catalog.<detail>.{short,body}` in EN and HU (a contract test checks it;
|
||||
the group is not called `detail`, which `actionErrors.contract` reads as a raw `HttpError.detail`).
|
||||
"""
|
||||
|
||||
# Two worker-authored strings that this module owns because both are coupled to `_RULES`.
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import NamedTuple
|
||||
|
||||
from app.downloads.adapters import (
|
||||
FACEBOOK_PARSE_WALL,
|
||||
POST_NOT_SHOWN_MESSAGE,
|
||||
POST_WITHOUT_VIDEO_MESSAGE,
|
||||
facebook_group_post_id,
|
||||
)
|
||||
from app.downloads.credentials import host_key_for_url
|
||||
|
||||
# Worker-authored strings that this module owns because each is coupled to `_RULES`.
|
||||
#
|
||||
# QUOTA_FULL_MESSAGE is raised by the worker and must classify back to `quota_exceeded`; its rule
|
||||
# below matches a fragment of it, and test_download_errors pins the pair, because a message and its
|
||||
# rule drifting apart is exactly how a category silently becomes unreachable.
|
||||
#
|
||||
# NETWORK_REDACTED_MESSAGE is the opposite direction: it REPLACES a message after classification, so
|
||||
# it deliberately has no rule. Nothing re-classifies a stored `job.error` today; if anything ever
|
||||
# does, it must carry the original code rather than re-deriving one from this text.
|
||||
# NETWORK_REDACTED_MESSAGE REPLACES a message after classification (`user_error_text`). It is written
|
||||
# to the shared asset too, and a second job that inherits that failed asset re-diagnoses the stored
|
||||
# text (`worker._process_job`), so it has a rule of its own that gives the category back.
|
||||
#
|
||||
# Other worker-raised texts are still inline literals at their raise sites. The ffmpeg ones and
|
||||
# "produced no output file" have markers here; the urlguard refusal deliberately has none and
|
||||
# classifies as `unknown` — it is a refusal, not a diagnosis, and no category would say more.
|
||||
#
|
||||
# A source adapter's verdicts on a link (`SourceAdapter.resolve_url`, E11.5 S4) are adapters.py's
|
||||
# constants, imported above; each has a rule below that gives its category back, and a catalog entry
|
||||
# that gives the site's sentence.
|
||||
QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
|
||||
NETWORK_REDACTED_MESSAGE = "The server could not reach that address."
|
||||
|
||||
@@ -30,6 +61,12 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
# Our own refusal: the worker found the user already at their storage cap (SEC-13). Placed
|
||||
# first — it is our wording, so it can never be a false match on an upstream message.
|
||||
("download storage is full", "quota_exceeded"),
|
||||
# Our own redaction of a transport failure (see NETWORK_REDACTED_MESSAGE) — our wording too.
|
||||
(NETWORK_REDACTED_MESSAGE.lower(), "network"),
|
||||
# Our own verdicts on a post link (adapters.py) — our wording too: its page holds no single video,
|
||||
# or the site answered the signed-in read with something that is not the post at all.
|
||||
(POST_WITHOUT_VIDEO_MESSAGE.lower(), "format_unavailable"),
|
||||
(POST_NOT_SHOWN_MESSAGE.lower(), "login_required"),
|
||||
# Login / age / members / bot-check — the user must be signed in (we deliberately are not).
|
||||
("sign in to confirm your age", "login_required"),
|
||||
("confirm your age", "login_required"),
|
||||
@@ -101,6 +138,120 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
)
|
||||
|
||||
|
||||
class Diagnosis(NamedTuple):
|
||||
"""What a failure was: the generic `code` (category) and, when a catalog entry matched, its
|
||||
host-specific `detail`."""
|
||||
|
||||
code: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Entry:
|
||||
"""One measured failure of one site. It matches when the download URL belongs to `host`
|
||||
(`credentials.SUPPORTED_HOSTS`), every one of `markers` is in the message, the cookie state is
|
||||
`cookie` (None: either), and — when `link` is given — it recognises the URL (a truthy result). A
|
||||
link predicate is the adapter's own, so the catalog and the download agree on what a link is."""
|
||||
|
||||
host: str
|
||||
markers: tuple[str, ...]
|
||||
code: str
|
||||
detail: str
|
||||
cookie: bool | None = None
|
||||
link: Callable[[str], object] | None = None
|
||||
|
||||
|
||||
_CATALOG: tuple[_Entry, ...] = (
|
||||
# Facebook answers a signed-out client with a page yt-dlp cannot parse — an 18+ reel, a closed
|
||||
# group's post (measured 2026-09-13: prod jobs 145, 156, 256 download once the owner's cookie is
|
||||
# attached). "Cannot parse data" is yt-dlp's generic wording for a broken extractor on EVERY site,
|
||||
# which is why the entry is scoped to Facebook and never becomes a general rule.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
||||
code="login_required",
|
||||
detail="facebook.login_wall",
|
||||
cookie=False,
|
||||
),
|
||||
# The same text WITH a cookie: an expired / signed-out cookie, or a page that also needs
|
||||
# impersonation (prod job 257). Telling this user to add cookies would be wrong — they have.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
||||
code="login_required",
|
||||
detail="facebook.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# A group post read with a cookie that Facebook answered with its login page (measured with a
|
||||
# refused cookie: no post id anywhere on the page) — the same refused cookie, found by the adapter.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=(POST_NOT_SHOWN_MESSAGE.lower(),),
|
||||
code="login_required",
|
||||
detail="facebook.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# A group post with no single video of its own. With a cookie the adapter reads the post's page
|
||||
# and says so itself (POST_WITHOUT_VIDEO_MESSAGE); without one yt-dlp is given the post link and
|
||||
# answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared
|
||||
# clips). With a cookie yt-dlp only ever gets the post's VIDEO link, so the same text there is about
|
||||
# that video, not the post — the category says it. Only for group post links (the adapter's own
|
||||
# test of one): a reel with no formats is a different failure.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),),
|
||||
code="format_unavailable",
|
||||
detail="facebook.group_post_video",
|
||||
link=facebook_group_post_id,
|
||||
),
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", "no video formats found"),
|
||||
code="format_unavailable",
|
||||
detail="facebook.group_post_video",
|
||||
cookie=False,
|
||||
link=facebook_group_post_id,
|
||||
),
|
||||
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
||||
# job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its
|
||||
# neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails
|
||||
# identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp).
|
||||
# The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so
|
||||
# "you're not a bot" with a straight quote never matches the real text.
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("[youtube]", "not a bot"),
|
||||
code="login_required",
|
||||
detail="youtube.bot_wall",
|
||||
cookie=False,
|
||||
),
|
||||
# The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first
|
||||
# one is being added now) — but "add cookies" is provably the wrong advice for someone who has,
|
||||
# and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no
|
||||
# longer valid. They have likely been rotated in the browser as a security measure."
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("[youtube]", "not a bot"),
|
||||
code="login_required",
|
||||
detail="youtube.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# The same gate one step later: the player response comes back, and then the CDN refuses the
|
||||
# stream itself ("unable to download video data: HTTP Error 403"). Measured 2026-09-16 on the
|
||||
# very same video, ~90 minutes after it was answering with the bot message — signed out it
|
||||
# 403'd on every attempt; with the owner's cookie attached it downloaded in full (519 MB). So
|
||||
# the category this text carries generically, `network`, is a lie here: nothing was wrong with
|
||||
# the network. Scoped to a cookie-less YouTube download, where that is what we measured.
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("unable to download video data", "403"),
|
||||
code="login_required",
|
||||
detail="youtube.media_forbidden",
|
||||
cookie=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def classify(msg: str | None) -> str:
|
||||
"""Return a category code for a raw failure message (``"unknown"`` if nothing matches)."""
|
||||
if not msg:
|
||||
@@ -112,6 +263,27 @@ def classify(msg: str | None) -> str:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def diagnose(
|
||||
msg: str | None, *, url: str | None = None, cookie_attached: bool = False
|
||||
) -> Diagnosis:
|
||||
"""The catalog entry for this failure of this link, else just its category (see the module doc).
|
||||
|
||||
`url` is the address yt-dlp was given; `cookie_attached` whether the acting user's cookie went
|
||||
with it. Without a URL only the category can be known."""
|
||||
host = host_key_for_url(url) if msg and url else None
|
||||
if host is not None:
|
||||
low = msg.lower() # type: ignore[union-attr] — `host` is only set when `msg` is truthy
|
||||
for e in _CATALOG:
|
||||
if (
|
||||
e.host == host
|
||||
and all(m in low for m in e.markers)
|
||||
and (e.cookie is None or e.cookie == cookie_attached)
|
||||
and (e.link is None or e.link(url)) # type: ignore[arg-type] — as `msg` above
|
||||
):
|
||||
return Diagnosis(e.code, e.detail)
|
||||
return Diagnosis(classify(msg))
|
||||
|
||||
|
||||
def user_error_text(source_kind: str, msg: str, code: str) -> str:
|
||||
"""The failure text to STORE on the job, given the raw message and its category.
|
||||
|
||||
|
||||
@@ -17,8 +17,134 @@ spec shape (all keys optional; defaults + browser-playability clamp applied by `
|
||||
`container`/`vcodec` keys survive only so an old stored profile deserializes without error.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import urllib.parse
|
||||
from collections.abc import Iterator
|
||||
|
||||
from app import logs
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# yt-dlp names what it is working on in these shapes, and each one is a row of someone's download
|
||||
# history: the source URL, the `[extractor] <id>:` prefix on nearly every warning and error, the
|
||||
# file path it writes to (`storage.rel_path` builds it from the channel and the title), a
|
||||
# playlist's title, and an id quoted mid-sentence. The job id is already on every line
|
||||
# (`logs.set_job` in the download child), so none of them adds anything an operator needs. What
|
||||
# stays: the host (it says which extractor is failing) and the extractor tag. Every pattern below
|
||||
# is a shape measured in yt-dlp's own source (2026.07.04), pinned in test_ytdlp_logging.py.
|
||||
_URL_RE = re.compile(r"https?://([^/\s]+)\S*")
|
||||
# `Destination:` is yt-dlp's own word in `[download] Destination: <path>`, not an id — the path
|
||||
# after it is what `_PATH_RE` takes out.
|
||||
_ID_PREFIX_RE = re.compile(r"(\[[\w:.-]+\] )(?!Destination:)[^\s:\[\]]+:")
|
||||
# The generic extractor's "id" is the URL's basename, percent-DECODED: `My%20Holiday.mp4` is
|
||||
# `My Holiday` — spaces, brackets, colons and all, which no pattern can bound (`Our Trip: Day 1`
|
||||
# ends at which `: `?). The job knows its own URL, so the exact name is taken out as a literal
|
||||
# (`hiding`). This pattern is the fallback for a line logged outside that scope: up to the first
|
||||
# `: `. Only this extractor: every other one names a site id, and letting the prefix run to `: `
|
||||
# everywhere would take the words out of lines like `[VideoConvertor] Converting …; Destination:`.
|
||||
_GENERIC_ID_RE = re.compile(r"(\[generic\] )(?!Destination:)[^\[\]\n]+?:(?= |$)", re.M)
|
||||
# `[youtube:tab] Playlist <title>: Downloading 5 items` — the title is not an id, and has spaces.
|
||||
_PLAYLIST_TITLE_RE = re.compile(r"(\] Playlist ).+?(?=: Downloading )")
|
||||
# `[download] Finished downloading playlist: <title>`, `[download] Downloading playlist: <title>`,
|
||||
# `… Skipping already downloaded playlist: <title>` — the title runs to the end of the line.
|
||||
_TITLE_TAIL_RE = re.compile(r"(playlist: |\[download\] Downloading \w+: ).*", re.I)
|
||||
# An id quoted mid-sentence, not as a prefix: `… not available for <id>`, and `--no-playlist`'s
|
||||
# `Downloading just the video <id>` (`_yes_playlist`) / `Downloading just video <id>` (multifeed).
|
||||
_BARE_ID_RE = re.compile(
|
||||
r"((?:subtitles not available for|Downloading just (?:the )?\w+) )[^\s,;]+"
|
||||
)
|
||||
# Two segments or more, and not glued to a word: `www.youtube.com/…` is what the URL pattern
|
||||
# leaves behind, and it must survive this one.
|
||||
_PATH_RE = re.compile(r"(?<![\w.…])/(?:[^/\s\"']+/)+[^/\s\"']*")
|
||||
|
||||
|
||||
_HIDDEN: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar(
|
||||
"formats_hidden", default=()
|
||||
)
|
||||
|
||||
|
||||
def generic_id(url: str) -> str:
|
||||
"""The name yt-dlp's generic extractor gives a URL: its basename, extension off, percent-
|
||||
decoded — `InfoExtractor._generic_id` (2026.07.04), pinned against it in the tests."""
|
||||
return urllib.parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0])
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def hiding(url: str | None) -> Iterator[None]:
|
||||
"""For the work on one URL: `log_safe` also takes out the name yt-dlp's generic extractor
|
||||
gives it, exactly, where yt-dlp puts it (`[generic] <name>:`). A context variable, so it
|
||||
holds for yt-dlp's logger calls too — they run on the same thread as the extraction."""
|
||||
name = generic_id(url) if url else ""
|
||||
token = _HIDDEN.set((*_HIDDEN.get(), name) if name else _HIDDEN.get())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_HIDDEN.reset(token)
|
||||
|
||||
|
||||
def log_safe(text: object) -> str:
|
||||
"""yt-dlp's text (its own lines, or a DownloadError's message) with the parts that name a
|
||||
person's download taken out. Every log line that carries yt-dlp's words goes through here,
|
||||
not just the adapter's: yt-dlp raises the same text it has just logged, so a line that skipped
|
||||
it would put back one line later exactly what the adapter took out."""
|
||||
text = str(text)
|
||||
for name in _HIDDEN.get():
|
||||
text = text.replace(f"[generic] {name}:", "[generic] …:")
|
||||
text = _URL_RE.sub(lambda m: f"{m.group(1)}/…", text)
|
||||
text = _ID_PREFIX_RE.sub(r"\1…:", text)
|
||||
text = _GENERIC_ID_RE.sub(r"\1…:", text)
|
||||
text = _PLAYLIST_TITLE_RE.sub(r"\1…", text)
|
||||
text = _TITLE_TAIL_RE.sub(r"\1…", text)
|
||||
text = _BARE_ID_RE.sub(r"\1…", text)
|
||||
return _PATH_RE.sub("…", text)
|
||||
|
||||
|
||||
class _YdlLog:
|
||||
"""yt-dlp's logger protocol, pointed at ours.
|
||||
|
||||
Without it yt-dlp explains itself to nobody: `quiet` sends its output to a stdout we do not
|
||||
read, so "Requested format is not available", a signature-extraction failure or a POT refusal
|
||||
reached us only as whatever text ended up in the raised exception. Measured against yt-dlp
|
||||
2026.07.04: with `logger` set it calls `debug` for extraction and progress lines and
|
||||
`warning`/`error` for the rest, and `quiet`/`no_warnings` do NOT gate those calls.
|
||||
|
||||
`debug` goes to TRACE: it is several lines per second during a download, so it shows only
|
||||
when the whole process runs at TRACE. (Not under a user's own Teljes consent — that lives in
|
||||
the API process's request scope, and the download child that runs yt-dlp has none.)
|
||||
|
||||
`error` goes to WARNING, not ERROR. yt-dlp calls it from `trouble()` just before it raises,
|
||||
on EVERY failed attempt — including the ones the pipeline then recovers from with its fallback
|
||||
client set. Whether the job failed is the pipeline's verdict (`download.job.failed`), and that
|
||||
is the one line whose level should say so."""
|
||||
|
||||
def _emit(self, level: int, msg: str) -> None:
|
||||
# The line goes as an ARGUMENT, not as the format string. yt-dlp writes progress as
|
||||
# `[download] 53.2% of 12.00MiB`, and while stdlib logging only applies `%` formatting to
|
||||
# a record that has args, this way the question never arises — and it is the shape ruff's
|
||||
# `G` rules ask for. (The sink's flood limiter groups a bare `%s` by the text itself, so
|
||||
# distinct yt-dlp warnings do not share one bucket — `logsink._template`.)
|
||||
log.log(level, "%s", log_safe(msg), extra=logs.event("download.ytdlp"))
|
||||
|
||||
def debug(self, msg):
|
||||
self._emit(logs.TRACE, msg)
|
||||
|
||||
def info(self, msg):
|
||||
self._emit(logging.DEBUG, msg)
|
||||
|
||||
def warning(self, msg):
|
||||
self._emit(logging.WARNING, msg)
|
||||
|
||||
def error(self, msg):
|
||||
self._emit(logging.WARNING, msg)
|
||||
|
||||
|
||||
_YDL_LOG = _YdlLog()
|
||||
|
||||
# Fields that affect the produced file — and therefore the cache identity.
|
||||
_SIG_KEYS = (
|
||||
@@ -57,6 +183,10 @@ _DEFAULTS = {
|
||||
# opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped
|
||||
# the v3 net — must re-derive through the current browser-safe guarantee.
|
||||
_SIG_VERSION = 4
|
||||
# Audio-only downloads version on their own, so a change to the audio bytes doesn't re-download every
|
||||
# cached video. v5: an extracted .mp3/.m4a is stored as itself instead of remuxed into an .mp4 that
|
||||
# iPad's WebKit refuses to play, so a cached remuxed audio asset must re-derive.
|
||||
_AUDIO_SIG_VERSION = 5
|
||||
|
||||
_SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"]
|
||||
|
||||
@@ -74,7 +204,13 @@ def normalize(spec: dict) -> dict:
|
||||
# rather than raising an unhandled 500 at enqueue.
|
||||
try:
|
||||
s["max_height"] = int(s["max_height"])
|
||||
except (TypeError, ValueError):
|
||||
except (TypeError, ValueError) as exc:
|
||||
# "best" is the opposite of what a height cap asks for: the user set a ceiling to save
|
||||
# bandwidth or disk, and the download quietly takes the largest rendition there is.
|
||||
log.warning(
|
||||
"the height cap on a download could not be read and was dropped",
|
||||
extra=logs.event("download.format.max_height_dropped", reason=type(exc).__name__),
|
||||
)
|
||||
s["max_height"] = None
|
||||
for b in ("embed_subs", "embed_chapters", "embed_thumbnail", "sponsorblock"):
|
||||
s[b] = bool(s[b])
|
||||
@@ -96,20 +232,34 @@ def normalize(spec: dict) -> dict:
|
||||
return s
|
||||
|
||||
|
||||
def format_sig(spec: dict, salt: str | None = None) -> str:
|
||||
def _sig_version(s: dict) -> int:
|
||||
return _AUDIO_SIG_VERSION if s["mode"] == "a" else _SIG_VERSION
|
||||
|
||||
|
||||
def format_sig(spec: dict, salt: str | None = None, *, version: int | None = None) -> str:
|
||||
"""Stable short hash of the output-affecting spec — the MediaAsset cache key component.
|
||||
|
||||
`salt` forks a distinct, non-dedup identity for the same spec (used by force-redownload so it
|
||||
always re-fetches into its own asset instead of sharing a cached file with other jobs)."""
|
||||
always re-fetches into its own asset instead of sharing a cached file with other jobs).
|
||||
`version` hashes under a signature version other than the current one (see `format_sigs`)."""
|
||||
s = normalize(spec)
|
||||
data = {k: s[k] for k in _SIG_KEYS}
|
||||
data["_v"] = _SIG_VERSION
|
||||
data["_v"] = _sig_version(s) if version is None else version
|
||||
if salt is not None:
|
||||
data["_salt"] = salt
|
||||
payload = json.dumps(data, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(payload.encode()).hexdigest()[:24]
|
||||
|
||||
|
||||
def format_sigs(spec: dict, salt: str | None = None) -> tuple[str, str]:
|
||||
"""`format_sig` under the current signature version and under the one before it. An asset keeps
|
||||
the signature it was created with, so a job queued before a deploy that bumped the version and
|
||||
run after it downloads into an asset carrying the previous one — which must still be recognised
|
||||
as that user's private asset (`credentials.is_authed_asset`), or it downloads without its cookie."""
|
||||
previous = _sig_version(normalize(spec)) - 1
|
||||
return format_sig(spec, salt), format_sig(spec, salt, version=previous)
|
||||
|
||||
|
||||
def build_ydl_opts(
|
||||
spec: dict,
|
||||
outtmpl: str,
|
||||
@@ -163,6 +313,12 @@ def build_ydl_opts(
|
||||
if extra:
|
||||
opts.update(extra)
|
||||
|
||||
# With a logger attached, `quiet` and `no_warnings` stop deciding whether yt-dlp explains itself
|
||||
# and only decide where — so both stay on above and the explanation comes here instead of to a
|
||||
# stdout nothing reads. AFTER the adapter merge, like `cookiefile` and the size cap: an adapter
|
||||
# must not be able to silence the one account of what went wrong that a failed job leaves.
|
||||
opts["logger"] = _YDL_LOG
|
||||
|
||||
# SEC-13: hard ceiling on a single download. AFTER the adapter merge, for the same reason
|
||||
# `cookiefile` is — a source adapter must not be able to raise a limit the quota layer set.
|
||||
#
|
||||
|
||||
@@ -16,7 +16,6 @@ Five passes each run:
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
@@ -24,14 +23,13 @@ from pathlib import Path
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.gc")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
||||
@@ -99,6 +97,7 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
if pause_s and touched % batch == 0:
|
||||
time.sleep(pause_s)
|
||||
|
||||
unreclaimed = 0
|
||||
for entry in root.iterdir():
|
||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
||||
continue
|
||||
@@ -111,7 +110,9 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
f.unlink()
|
||||
removed += 1
|
||||
except OSError:
|
||||
pass
|
||||
# silent-ok: counted, not logged per file — a broken mount would put one line
|
||||
# under every orphan. The total is reported once at the end of the sweep.
|
||||
unreclaimed += 1
|
||||
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
|
||||
for d in sorted(
|
||||
(p for p in entry.rglob("*") if p.is_dir()),
|
||||
@@ -120,12 +121,20 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
):
|
||||
try:
|
||||
d.rmdir()
|
||||
# silent-ok: `rmdir` refusing means the directory still holds files, which is
|
||||
# the ordinary answer here — this prune only takes the ones that emptied.
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
entry.rmdir()
|
||||
# silent-ok: the same prune one level up, at the channel directory.
|
||||
except OSError:
|
||||
pass
|
||||
if unreclaimed:
|
||||
log.warning(
|
||||
"some orphaned download files could not be reclaimed",
|
||||
extra=logs.event("download.gc.unreclaimed", files=unreclaimed, removed=removed),
|
||||
)
|
||||
return removed
|
||||
|
||||
|
||||
|
||||
@@ -49,6 +49,8 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
|
||||
exp_s, sig = grant.split(".", 1)
|
||||
try:
|
||||
exp = int(exp_s)
|
||||
# silent-ok: an untrusted grant that does not parse is simply not a valid grant, and the
|
||||
# honest answer to it — a password prompt — is the same as for one that expired.
|
||||
except ValueError:
|
||||
return False
|
||||
if exp < datetime.now(timezone.utc).timestamp():
|
||||
|
||||
@@ -66,6 +66,8 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
|
||||
width = (data[3] << 8) + data[4]
|
||||
return width, height
|
||||
f.seek(length - 2, 1) # skip this segment's payload
|
||||
# silent-ok: None omits width/height from the unfurl card, which is what the card does for
|
||||
# every poster whose dimensions were never asked for. Nothing is claimed that is not true.
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
@@ -8,9 +7,8 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
@@ -25,12 +23,13 @@ from app.downloads.jobdb import (
|
||||
_staging_dir,
|
||||
_touch_job,
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.models import DISPLAY_TEXT_MAX, DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
|
||||
log = logging.getLogger("siftlode.worker")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class _Aborted(Exception):
|
||||
@@ -99,6 +98,8 @@ def _parse_upload_date(raw) -> date | None:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(str(raw), "%Y%m%d").date()
|
||||
# silent-ok: a date we cannot read becomes a date the card does not show, which is what the
|
||||
# card shows for every source that never supplies one. Absent is the honest rendering.
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@@ -114,9 +115,10 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
# `or info.get("id")` avoids a blank card while downloading when the cleaned title is empty
|
||||
# (e.g. a title that was only a social engagement prefix); the completion path later sets
|
||||
# the final title with its own source_ref fallback.
|
||||
asset.title = normalize_title(info.get("title")) or info.get("id")
|
||||
adapter = adapters.resolve_adapter(None, None, info)
|
||||
asset.title = normalize_title(adapter.title(info)) or info.get("id")
|
||||
asset.uploader = info.get("uploader") or info.get("channel")
|
||||
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
|
||||
asset.uploader_url = adapter.uploader_url(info)
|
||||
asset.thumbnail_url = info.get("thumbnail")
|
||||
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
||||
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
||||
@@ -138,7 +140,14 @@ def _make_progress_hook(job_id: int, asset_id: int):
|
||||
try:
|
||||
_fill_asset_meta_early(asset_id, d["info_dict"])
|
||||
except Exception: # noqa: BLE001 — metadata is best-effort, never fail a download
|
||||
pass
|
||||
# Best-effort is about the DOWNLOAD, not about knowing. What this swallows is
|
||||
# anything from a shape change in yt-dlp's info dict to the database being down,
|
||||
# and all the user sees is a card with an id on it until the job finishes.
|
||||
log.info(
|
||||
"early metadata could not be filled in for this download",
|
||||
extra=logs.event("download.meta.early_fill_failed", asset_id=asset_id),
|
||||
exc_info=True,
|
||||
)
|
||||
if d.get("status") != "downloading":
|
||||
return
|
||||
if not allow_write():
|
||||
@@ -299,7 +308,7 @@ def _produced_file(info: dict, staging: Path) -> Path:
|
||||
and not _YTDLP_TEMP_RE.search(p.name)
|
||||
]
|
||||
if not cands:
|
||||
raise RuntimeError("yt-dlp produced no output file")
|
||||
raise StatedFailure("yt-dlp produced no output file")
|
||||
return max(cands, key=lambda p: p.stat().st_size)
|
||||
|
||||
|
||||
@@ -323,14 +332,33 @@ def _extract_with_fallback(
|
||||
):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt.
|
||||
error); a Facebook download WITH a cookie is retried as Chrome behind "Cannot parse data"; other
|
||||
sources make a single attempt. Returns the info dict of the successful attempt.
|
||||
|
||||
Before the first attempt the adapter may resolve `url` to the link that really holds the video (a
|
||||
Facebook group post → its video's watch link); when the page shows no single video there, or is not
|
||||
the post at all (a refused cookie's login page), the adapter raises its own verdict
|
||||
(adapters.POST_*_MESSAGE) instead of letting yt-dlp guess, and a failed page read fails the
|
||||
download as itself.
|
||||
|
||||
`cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated.
|
||||
|
||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt."""
|
||||
client_sets = adapter.client_sets()
|
||||
client_sets = adapter.client_sets(authed=cookiefile is not None)
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
# Resolving may read a page first — network time the stall watchdog must not see as a stall.
|
||||
_touch_job(job_id)
|
||||
target = adapter.resolve_url(
|
||||
url,
|
||||
formats.build_ydl_opts(
|
||||
spec,
|
||||
outtmpl,
|
||||
_make_progress_hook(job_id, asset_id),
|
||||
cookiefile=cookiefile,
|
||||
max_bytes=max_bytes,
|
||||
),
|
||||
)
|
||||
last_exc: Exception | None = None
|
||||
for i, clients in enumerate(client_sets):
|
||||
opts = formats.build_ydl_opts(
|
||||
@@ -348,7 +376,7 @@ def _extract_with_fallback(
|
||||
# killed — acceptable, as socket_timeout + extractor_retries bound one attempt well under it.)
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||
return ydl.extract_info(url, download=True)
|
||||
return ydl.extract_info(target, download=True)
|
||||
except yt_dlp.utils.DownloadError as exc:
|
||||
last_exc = exc
|
||||
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||
@@ -356,13 +384,22 @@ def _extract_with_fallback(
|
||||
"job %s: clients %s failed (%s); retrying with fallback",
|
||||
job_id,
|
||||
clients,
|
||||
str(exc)[:70],
|
||||
formats.log_safe(exc)[:70],
|
||||
)
|
||||
for p in list(staging.iterdir()): # drop partials before the retry
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
except OSError as exc:
|
||||
# A partial left behind is not inert: the next client set resumes into it,
|
||||
# so the retry this loop exists to make clean starts dirty instead.
|
||||
log.info(
|
||||
"a partial file could not be removed before the retry",
|
||||
extra=logs.event(
|
||||
"download.retry.partial_unlink_failed",
|
||||
job_id=job_id,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
_set_job(job_id, progress=0, phase="downloading")
|
||||
continue
|
||||
raise
|
||||
@@ -385,8 +422,12 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
or just remux when the codecs are fine but the container isn't. **Scope:** only the mp4 compat
|
||||
profile is normalized; an explicit `vcodec` or a non-mp4 custom profile is the user opting out
|
||||
of the guarantee and is left as-is (so a `webm` profile can still ship VP9/Opus). Returns the
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk."""
|
||||
if spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk.
|
||||
|
||||
An audio download (mode "a") is never touched: yt-dlp already extracted it to .mp3 / .m4a, which
|
||||
every browser plays as audio/mpeg / audio/mp4. Remuxing it into .mp4 served an mp3 as video/mp4,
|
||||
which iPad's WebKit refuses to play (prod job 268)."""
|
||||
if spec.get("mode") == "a" or spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
return produced
|
||||
vcodec, acodec = editmod.probe_codecs(produced)
|
||||
is_mp4 = produced.suffix.lower() == ".mp4"
|
||||
@@ -425,6 +466,7 @@ def _download(
|
||||
) -> None:
|
||||
staging = _staging_dir(asset_id)
|
||||
cookiefile: Path | None = None
|
||||
url: str | None = None # read by the failure branch, which can run before it is resolved
|
||||
try:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
@@ -440,17 +482,15 @@ def _download(
|
||||
try:
|
||||
urlguard.check_url(url)
|
||||
except urlguard.BlockedUrl as e:
|
||||
raise RuntimeError(f"Refusing to download from this address ({e.reason}).") from e
|
||||
raise StatedFailure(f"Refusing to download from this address ({e.reason}).") from e
|
||||
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
remaining_bytes: int | None = None
|
||||
with SessionLocal() as cdb:
|
||||
asset_sig = cdb.execute(
|
||||
select(MediaAsset.format_sig).where(MediaAsset.id == asset_id)
|
||||
).scalar_one_or_none()
|
||||
if asset_sig is not None:
|
||||
target = cdb.get(MediaAsset, asset_id)
|
||||
if target is not None:
|
||||
cookiefile = credentials.resolve_cookiefile(
|
||||
cdb, user_id, url, asset_sig, spec, staging
|
||||
cdb, user_id, url, target, spec, staging
|
||||
)
|
||||
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
|
||||
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a
|
||||
@@ -484,13 +524,19 @@ def _download(
|
||||
produced = _produced_file(info, staging)
|
||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||
ext = produced.suffix.lstrip(".").lower()
|
||||
# Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's
|
||||
# `info` dict, whose fields are absent for some sources (Facebook among them): deriving
|
||||
# media_kind from those nulls misclassified a real H.264 video as a download-only 'file', and a
|
||||
# missing duration hid the trim action. Probed before the sidecars so the .nfo runtime has it
|
||||
# too. Same probe the upload ingest uses.
|
||||
vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
|
||||
meta = storage.MediaMeta(
|
||||
video_id=info.get("id") or source_ref,
|
||||
title=normalize_title(info.get("title")) or source_ref,
|
||||
title=normalize_title(adapter.title(info)) or source_ref,
|
||||
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
||||
upload_date=_parse_upload_date(info.get("upload_date")),
|
||||
uniqueid_type=adapter.nfo_uniqueid_type(),
|
||||
duration_s=int(info["duration"]) if info.get("duration") else None,
|
||||
duration_s=duration_s,
|
||||
description=info.get("description"),
|
||||
thumbnail_url=info.get("thumbnail"),
|
||||
)
|
||||
@@ -516,12 +562,11 @@ def _download(
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = size
|
||||
asset.duration_s = meta.duration_s
|
||||
# Codecs/dims come from a PROBE of the finished file — the ground truth — NOT yt-dlp's
|
||||
# `info` dict, whose codec fields are absent for some sources (Facebook among them):
|
||||
# deriving media_kind from those nulls misclassified a real H.264 video as a
|
||||
# download-only 'file' with no in-app play/edit. Same probe the upload ingest uses.
|
||||
asset.vcodec, asset.acodec, asset.width, asset.height = probe.resolve_media_facts(
|
||||
final, info
|
||||
asset.vcodec, asset.acodec, asset.width, asset.height = (
|
||||
vcodec,
|
||||
acodec,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
asset.container = ext
|
||||
asset.media_kind = mediakind.derive(asset.vcodec, asset.acodec)
|
||||
@@ -552,20 +597,30 @@ def _download(
|
||||
}
|
||||
if job and not job.display_name and meta.title:
|
||||
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
||||
fields["display_name"] = meta.title[:255]
|
||||
fields["display_name"] = meta.title[:DISPLAY_TEXT_MAX]
|
||||
# Atomic terminal write: finalizes ONLY while the row is still running, so a child that
|
||||
# finishes just after a route cancel/pause can't resurrect it (see finalize_if_running).
|
||||
finalized = service.finalize_if_running(db, job_id, **fields)
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, meta.title, job_id, kind="download")
|
||||
log.info("job %s done: %s", job_id, rel)
|
||||
# The asset id, not `rel`: the path is built from the channel and the title.
|
||||
log.info(
|
||||
"job %s done: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event("download.job.done", job_id=job_id, asset_id=asset_id),
|
||||
)
|
||||
|
||||
except (_Aborted, editmod.EditAborted):
|
||||
# Paused/canceled mid-download or mid-transcode: free the asset for a later retry; keep the
|
||||
# job's status (the route set it to paused/canceled). ref_count is adjusted by the route.
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info("job %s aborted (pause/cancel)", job_id)
|
||||
log.info(
|
||||
"job %s aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("download.job.aborted", job_id=job_id),
|
||||
)
|
||||
except _OwnerRefused as exc:
|
||||
# Fail this job only. The asset is never marked `error`; it goes back to `pending` (below),
|
||||
# so another user's job for the same video is claimed and downloaded normally instead of
|
||||
@@ -578,6 +633,7 @@ def _download(
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
error_detail=None,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
@@ -585,29 +641,41 @@ def _download(
|
||||
# it, and this one must already be out of the way. (The staging tree is still torn down in
|
||||
# `finally` below — the same narrow window the pause/cancel path above has always had.)
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info("job %s refused for its owner: %s", job_id, msg)
|
||||
log.info(
|
||||
"job %s refused for its owner: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event("download.job.refused", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
|
||||
msg = str(exc)[:500]
|
||||
code = dlerrors.classify(msg)
|
||||
# The catalog needs the link and whether the user's cookie went with it: the same yt-dlp
|
||||
# text is Facebook's login wall without one and a refused cookie with one.
|
||||
diagnosis = dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
|
||||
# SEC-12: redact a transport-level readout for user-supplied URLs (see user_error_text).
|
||||
msg = dlerrors.user_error_text(source_kind, msg, code)
|
||||
msg = dlerrors.user_error_text(source_kind, msg, diagnosis.code)
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
|
||||
# with `error` (error also holds the asset → would re-open the double-release class).
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never
|
||||
# overwritten with `error` (error also holds the asset → would re-open the
|
||||
# double-release class).
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=code,
|
||||
error_code=diagnosis.code,
|
||||
error_detail=diagnosis.detail,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("job %s failed: %s", job_id, msg)
|
||||
# After the write: a job the write could not fail is not a failed job. When it raises,
|
||||
# the runner reports this failure with it (`runner._run`) and the worker fails the job.
|
||||
# silent-ok: not silent — `_log_download_failed` is the report.
|
||||
_log_download_failed(job_id, msg, diagnosis.code, exc)
|
||||
finally:
|
||||
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
|
||||
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
|
||||
@@ -616,7 +684,50 @@ def _download(
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
def _failure_stack(exc: BaseException) -> dict[str, str]:
|
||||
"""The `extra=` fragment that puts a failure's traceback on its line — or nothing.
|
||||
|
||||
Nothing for yt-dlp's DownloadError and for our own `StatedFailure` (`ffmpeg failed (1): …`):
|
||||
their message is the whole story, and their stack is only the `except` that caught them.
|
||||
Everything else keeps it — a TypeError on an odd info dict, a plain RuntimeError, an OSError,
|
||||
a database error: that is our code or our platform failing, and the stack is the lead.
|
||||
|
||||
Scrubbed, link by link (`logs.scrubbed_stack`): a stack's messages are data — an OSError names
|
||||
its file, and a database error raised inside `except DownloadError` carries yt-dlp's
|
||||
unscrubbed text as its context."""
|
||||
if isinstance(exc, (yt_dlp.utils.DownloadError, StatedFailure)):
|
||||
return {}
|
||||
return logs.scrubbed_stack(exc, formats.log_safe)
|
||||
|
||||
|
||||
def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseException) -> None:
|
||||
"""The child's `download.job.failed` line. `msg` is the user's copy and keeps their URL; the
|
||||
log's copy goes through `log_safe`, or it would put back one line later the URL and id the
|
||||
yt-dlp adapter took out of the line before it. The stack only when the classifier could not
|
||||
place the failure: a recognised one is fully described by its code."""
|
||||
unplaced = code in (None, "", "unknown")
|
||||
log.warning(
|
||||
"job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra={
|
||||
**logs.event(
|
||||
"download.job.failed",
|
||||
job_id=job_id,
|
||||
error_code=code,
|
||||
error_type=type(exc).__name__,
|
||||
),
|
||||
**(_failure_stack(exc) if unplaced else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None = None) -> None:
|
||||
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
|
||||
where its line and its event code live. `exc` is the exception when there was one.
|
||||
|
||||
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
|
||||
Logged after the write, as on the download error path."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
@@ -632,7 +743,22 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("edit job %s failed: %s", job_id, msg)
|
||||
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
||||
# built from the channel and the title.
|
||||
log.warning(
|
||||
"edit job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra={
|
||||
**logs.event(
|
||||
"edit.job.failed",
|
||||
job_id=job_id,
|
||||
asset_id=asset_id,
|
||||
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
||||
),
|
||||
**(_failure_stack(exc) if exc is not None else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
@@ -684,7 +810,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
)
|
||||
if not dest_staging.exists():
|
||||
raise RuntimeError("ffmpeg produced no output file")
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
|
||||
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||
storage.place_file(dest_staging, settings.download_root, rel)
|
||||
@@ -720,13 +846,24 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, dl_title, job_id, kind="edit")
|
||||
log.info("edit job %s done: %s", job_id, rel)
|
||||
# The asset id, not `rel`: the path is built from the channel and the title.
|
||||
log.info(
|
||||
"edit job %s done: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event("edit.job.done", job_id=job_id, asset_id=asset_id),
|
||||
)
|
||||
|
||||
except editmod.EditAborted:
|
||||
# Paused/canceled mid-encode: free the asset for a later retry (ref_count is the route's job).
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info("edit job %s aborted (pause/cancel)", job_id)
|
||||
log.info(
|
||||
"edit job %s aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("edit.job.aborted", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500])
|
||||
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
@@ -64,6 +64,8 @@ def _facts_from_probe(
|
||||
raw = fmt.get("duration")
|
||||
if raw is not None:
|
||||
duration = int(float(raw))
|
||||
# silent-ok: an unusable duration is stored as no duration, which the UI renders as unknown
|
||||
# rather than as zero — the honest state, and the one a re-probe can still fill in.
|
||||
except (TypeError, ValueError):
|
||||
duration = None
|
||||
return vcodec, acodec, width, height, duration
|
||||
@@ -81,10 +83,12 @@ def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | N
|
||||
|
||||
def resolve_media_facts(
|
||||
path: Path, info: dict
|
||||
) -> tuple[str | None, str | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height) for the stored download asset: PROBE the finished file on disk
|
||||
(the ground truth). yt-dlp's `info` omits codec/dimension fields for some sources (Facebook among
|
||||
them), so deriving media_kind from it there labelled a real H.264 video a download-only 'file'.
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) for the stored download asset: PROBE the finished
|
||||
file on disk (the ground truth). yt-dlp's `info` omits codec/dimension fields for some sources
|
||||
(Facebook among them), so deriving media_kind from it there labelled a real H.264 video a
|
||||
download-only 'file'. It omits the duration too (a Facebook reel, an Instagram reel), and a NULL
|
||||
duration hides the Library row's trim action although the file plays for minutes.
|
||||
|
||||
Precedence is by whether the probe RAN, not per-field: if ffprobe read the file, its codec verdict
|
||||
is authoritative — INCLUDING a deliberate None (cover-art-only audio, a still image). A per-field
|
||||
@@ -92,16 +96,23 @@ def resolve_media_facts(
|
||||
have (yt-dlp reports a progressive `vcodec` for an audio-mode download whose video it then strips),
|
||||
mislabelling an audio download as a broken in-app 'video'. Only when ffprobe could NOT read the
|
||||
file at all do we fall back to `info` (the pre-probe behaviour, a safety net); the cosmetic
|
||||
dimensions still gap-fill from `info` in either case."""
|
||||
dimensions and the duration still gap-fill from `info` in either case."""
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
info_v = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||
info_a = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
info_w = int(info["width"]) if info.get("width") else None
|
||||
info_h = int(info["height"]) if info.get("height") else None
|
||||
info_d = int(info["duration"]) if info.get("duration") else None
|
||||
if data is None:
|
||||
# ffprobe could not read the file — fall back to the extractor's info entirely.
|
||||
return info_v, info_a, info_w, info_h
|
||||
pv, pa, pw, ph, _dur = _facts_from_probe(data)
|
||||
return info_v, info_a, info_w, info_h, info_d
|
||||
pv, pa, pw, ph, pd = _facts_from_probe(data)
|
||||
# Probe ran → its codecs win outright (a None means "no such track", not "unknown"); info only
|
||||
# gap-fills the cosmetic width/height the probe couldn't read.
|
||||
return pv, pa, pw if pw is not None else info_w, ph if ph is not None else info_h
|
||||
# gap-fills the width/height/duration the probe couldn't read.
|
||||
return (
|
||||
pv,
|
||||
pa,
|
||||
pw if pw is not None else info_w,
|
||||
ph if ph is not None else info_h,
|
||||
pd if pd is not None else info_d,
|
||||
)
|
||||
|
||||
@@ -11,18 +11,20 @@ writes the terminal job/asset state itself (done / error, or a cooperative pause
|
||||
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from app import logs
|
||||
from app import logsink
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats
|
||||
from app.models import DownloadJob
|
||||
|
||||
log = logging.getLogger("siftlode.worker.runner")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _deprioritize() -> None:
|
||||
@@ -52,12 +54,21 @@ def _deprioritize() -> None:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logs.configure_logging("runner")
|
||||
_deprioritize()
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: python -m app.downloads.runner <job_id>", file=sys.stderr)
|
||||
# noqa T201: a usage line is for the person who typed the command. Logging is already
|
||||
# configured here, and in a deployment it writes JSON records — the one reader of this
|
||||
# line wants the bare sentence, not a record with a request id they do not have.
|
||||
print( # noqa: T201
|
||||
"usage: python -m app.downloads.runner <job_id>", file=sys.stderr
|
||||
)
|
||||
raise SystemExit(2)
|
||||
job_id = int(sys.argv[1])
|
||||
# This whole process exists to run ONE job — tag every line it prints with it, so the child's
|
||||
# output joins up with the worker's and the API's view of the same download.
|
||||
logs.set_job(job_id)
|
||||
logsink.start() # flushed at exit (atexit) — this process has no shutdown hook of its own
|
||||
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
@@ -71,14 +82,40 @@ def main() -> None:
|
||||
if asset_id is None:
|
||||
return
|
||||
|
||||
_run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id)
|
||||
|
||||
|
||||
def _run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id) -> None:
|
||||
"""Run the job, and be the last stop before Python's own exception hook.
|
||||
|
||||
What gets past the pipeline is a failure it could not even record (the database refused the
|
||||
write that fails the job). Python's hook would print it to stderr raw — and its context is the
|
||||
download's own failure, whose text is the URL or ffmpeg's stderr naming the file. Here it is
|
||||
one line with the whole chain scrubbed; the non-zero exit then has the worker fail the job
|
||||
(`cause=child_exit`)."""
|
||||
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
||||
# real work, and to keep the import graph obvious: runner is a thin entry over the pipeline.
|
||||
from app.downloads.pipeline import _download, _process_edit
|
||||
|
||||
if job_kind == "edit":
|
||||
_process_edit(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||
with formats.hiding(source_ref):
|
||||
try:
|
||||
if job_kind == "edit":
|
||||
_process_edit(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
||||
# noqa TRY400: the stack goes with it scrubbed (`scrubbed_stack`); `exc_info` is raw.
|
||||
log.error( # noqa: TRY400
|
||||
"the download child for job %s failed",
|
||||
job_id,
|
||||
extra={
|
||||
**logs.event(
|
||||
"download.child.failed", job_id=job_id, error_type=type(exc).__name__
|
||||
),
|
||||
**logs.scrubbed_stack(exc, formats.log_safe),
|
||||
},
|
||||
)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -7,6 +7,7 @@ handles everything else. State transitions used by the routes (pause/resume/canc
|
||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
@@ -19,7 +20,7 @@ from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobStatus
|
||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||
from app.models import DISPLAY_TEXT_MAX, Channel, DownloadJob, MediaAsset, Video
|
||||
|
||||
|
||||
class EditError(Exception):
|
||||
@@ -80,11 +81,15 @@ def get_or_create_asset(
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
try:
|
||||
db.flush()
|
||||
# A SAVEPOINT: losing the race undoes only this insert, never the caller's own pending writes
|
||||
# (Retry clears the job's error before it gets here; a whole-session rollback threw that away).
|
||||
with db.begin_nested():
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
# silent-ok: the expected outcome of the race this savepoint exists for — two jobs wanting the
|
||||
# same asset. Losing it means the other insert won, and re-selecting is the whole handling.
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
asset = db.execute(stmt).scalar_one()
|
||||
return asset
|
||||
|
||||
@@ -174,7 +179,7 @@ def enqueue(
|
||||
profile_id=profile_id,
|
||||
profile_snapshot=spec,
|
||||
display_name=display_name
|
||||
or (asset.title[:255] if asset.status == "ready" and asset.title else None),
|
||||
or (asset.title[:DISPLAY_TEXT_MAX] if asset.status == "ready" and asset.title else None),
|
||||
status="queued",
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
@@ -193,10 +198,13 @@ def enqueue(
|
||||
|
||||
|
||||
def _clip_title(source_title: str | None, display_name: str | None) -> str:
|
||||
# Clipped to DISPLAY_TEXT_MAX, not to a literal: this feeds `DownloadJob.display_name` through
|
||||
# `enqueue_edit`, the same column `display_text` guards. A machine-made title, so it is clipped
|
||||
# rather than run through `display_text` — blank here means "no title", not None.
|
||||
if display_name:
|
||||
return display_name[:255]
|
||||
return display_name[:DISPLAY_TEXT_MAX]
|
||||
base = source_title or "Clip"
|
||||
return f"{base} (clip)"[:255]
|
||||
return f"{base} (clip)"[:DISPLAY_TEXT_MAX]
|
||||
|
||||
|
||||
def enqueue_edit(
|
||||
@@ -250,7 +258,7 @@ def enqueue_edit(
|
||||
source_asset_id=src_asset.id,
|
||||
edit_spec=spec,
|
||||
profile_snapshot={},
|
||||
display_name=display_name or asset.title,
|
||||
display_name=display_name or asset.title[:DISPLAY_TEXT_MAX],
|
||||
status="queued",
|
||||
queue_pos=next_queue_pos(db),
|
||||
)
|
||||
@@ -341,13 +349,20 @@ def pause(db: Session, job: DownloadJob) -> None:
|
||||
def resume(db: Session, job: DownloadJob) -> None:
|
||||
"""Requeue a paused/errored job for a fresh attempt, clearing its error. If the SHARED asset itself
|
||||
failed, reset it to pending so the worker actually re-downloads — otherwise the requeued job would
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset)."""
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset).
|
||||
|
||||
A cookie the user added AFTER the job was queued moves the job onto their private asset first
|
||||
(`_follow_the_cookie`): the worker only ever attaches a cookie to that identity, so without the move
|
||||
the Retry a login-wall message asks for would fail on the same wall."""
|
||||
if job.status in (JobStatus.PAUSED, JobStatus.ERROR):
|
||||
job.progress = 0
|
||||
job.error = None
|
||||
job.error_code = None
|
||||
job.error_detail = None
|
||||
if job.asset_id:
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
if asset is not None:
|
||||
asset = _follow_the_cookie(db, job, asset)
|
||||
if asset and asset.status == "error": # MediaAsset.status (no enum), not a JobStatus
|
||||
asset.status = "pending"
|
||||
asset.error = None
|
||||
@@ -355,6 +370,32 @@ def resume(db: Session, job: DownloadJob) -> None:
|
||||
db.commit()
|
||||
|
||||
|
||||
def _follow_the_cookie(db: Session, job: DownloadJob, asset: MediaAsset) -> MediaAsset:
|
||||
"""Re-point a requeued job at the asset enqueue would pick NOW, when the user has a cookie for the
|
||||
link's host but the job still holds an asset the worker won't attach it to (queued before the
|
||||
cookie existed). Returns the asset the job holds afterwards.
|
||||
|
||||
Only a `pending` / `error` asset moves — neither has a file nor a download in flight. A `ready` one
|
||||
already has the file, and a `downloading` one may be this job's own child still being stopped by
|
||||
the pause; the next Retry after it settles moves the job then. Removing a cookie never moves a job
|
||||
back: the worker attaches nothing without a credential, so the private asset is as good as any."""
|
||||
if asset.status not in ("pending", "error"):
|
||||
return asset
|
||||
url = source_url(job.source_kind, job.source_ref)
|
||||
salt = credentials.salt_for_enqueue(db, job.user_id, url)
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
if salt is None or credentials.is_authed_asset(asset, spec, job.user_id, url):
|
||||
return asset
|
||||
target = get_or_create_asset(
|
||||
db, job.source_kind, job.source_ref, formats.format_sig(spec, salt=salt)
|
||||
)
|
||||
populate_from_catalog(db, target)
|
||||
release_asset(db, job) # drop the old hold BEFORE re-pointing — it keys off job.asset_id
|
||||
incr_ref_count(db, target.id)
|
||||
job.asset_id = target.id
|
||||
return target
|
||||
|
||||
|
||||
def cancel(db: Session, job: DownloadJob) -> None:
|
||||
"""Cancel a job unless it's already done/canceled, releasing its asset hold WHILE it still counts
|
||||
as holding — `canceled` ∉ HOLDING, so a later delete won't double-release."""
|
||||
@@ -373,13 +414,31 @@ def delete(db: Session, job: DownloadJob) -> None:
|
||||
|
||||
def redownload(db: Session, job: DownloadJob) -> None:
|
||||
"""Rebuild a finished/failed job's file into a FRESH, private asset (the cache sig is salted with
|
||||
the job id + current asset id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
the new asset's own id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
so any public share link (DownloadLink -> job_id) keeps resolving to the newly-downloaded file.
|
||||
The old hold is dropped first (freeing the old file iff this job was its last holder). The caller
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409)."""
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409).
|
||||
|
||||
When the user has a cookie for the link's host, the salt carries it (`credentials.rebuild_salt`),
|
||||
so the worker recognises the rebuild as their private identity and attaches the cookie — a cookie
|
||||
download, or one the user added after a failure, rebuilds with the cookie."""
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
sig = formats.format_sig(spec, salt=f"{job.id}:{job.asset_id}")
|
||||
asset = get_or_create_asset(db, job.source_kind, job.source_ref, sig)
|
||||
cookie = credentials.salt_for_enqueue(
|
||||
db, job.user_id, source_url(job.source_kind, job.source_ref)
|
||||
)
|
||||
# The salt names the row's own id, which exists only once the row does: insert it under a unique
|
||||
# placeholder, then write the real signature in the same transaction (nobody sees the placeholder).
|
||||
asset = MediaAsset(
|
||||
source_kind=job.source_kind,
|
||||
source_ref=job.source_ref,
|
||||
format_sig=f"rebuild:{uuid.uuid4().hex}",
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
asset.format_sig = formats.format_sig(spec, salt=credentials.rebuild_salt(asset.id, cookie))
|
||||
db.flush()
|
||||
populate_from_catalog(db, asset)
|
||||
# Drop the old hold BEFORE re-pointing the job — release_asset keys off the job's current asset.
|
||||
release_asset(db, job)
|
||||
@@ -389,6 +448,8 @@ def redownload(db: Session, job: DownloadJob) -> None:
|
||||
job.progress = 0
|
||||
job.phase = None
|
||||
job.error = None
|
||||
job.error_code = None # a rebuild that fails again gets its own diagnosis, never the old one
|
||||
job.error_detail = None
|
||||
job.speed_bps = None
|
||||
job.eta_s = None
|
||||
# Re-derive the display name from the freshly cleaned title on completion (else an older, un-cleaned
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""The failures the download pipeline raises itself, with the whole story in the message.
|
||||
|
||||
`ffmpeg failed (1): …`, `yt-dlp produced no output file`, a post that shows no video: the message
|
||||
says what happened, and the stack is only the `except` that caught it. A failure line leaves that
|
||||
stack off (`pipeline._failure_stack`) — but ONLY for these. A plain `RuntimeError` is also what
|
||||
Python raises for a bug of ours (a dict changed size during iteration, a generator's
|
||||
StopIteration, `RecursionError`, `NotImplementedError`), and that one's stack is the only lead.
|
||||
"""
|
||||
|
||||
|
||||
class StatedFailure(RuntimeError):
|
||||
"""A failure we raised on purpose, whose message is its full explanation. A `RuntimeError` so
|
||||
every `except RuntimeError` that caught these before still does."""
|
||||
@@ -19,8 +19,11 @@ from datetime import date
|
||||
from pathlib import Path
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
from app import logs
|
||||
from app.downloads import mediakind
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Reserve room for the "_id.ext" suffix + parent dirs within the 255-char component limit.
|
||||
_MAX_TITLE = 120
|
||||
# User-facing (Content-Disposition) names can be longer than an on-disk component.
|
||||
@@ -294,7 +297,13 @@ def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Pat
|
||||
if thumb_src and thumb_src.exists():
|
||||
shutil.copyfile(str(thumb_src), str(_sidecar(media, ".jpg")))
|
||||
return True
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# The download itself still completes, so the job reads as done while the library it was
|
||||
# downloaded FOR has no .nfo to read and no poster to show.
|
||||
log.warning(
|
||||
"the sidecars beside a download could not be written",
|
||||
extra=logs.event("download.sidecars.write_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -304,14 +313,23 @@ def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
except FileNotFoundError:
|
||||
pass # silent-ok: already gone is the outcome this function wanted
|
||||
except OSError as exc:
|
||||
# The row goes and the bytes stay. Nothing else ever revisits this path, so the disk
|
||||
# quietly keeps a file no account has any way to see, let alone delete.
|
||||
log.info(
|
||||
"a deleted download left a file behind",
|
||||
extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__),
|
||||
)
|
||||
# Prune empty Season/Channel dirs up to (but not including) the root.
|
||||
root = Path(download_root).resolve()
|
||||
parent = media.parent
|
||||
while parent != root and root in parent.parents:
|
||||
try:
|
||||
parent.rmdir() # only succeeds if empty
|
||||
# silent-ok: this `rmdir` failing is the loop's stop condition — the parent still
|
||||
# holds another download's files, so the prune has gone as far up as it may.
|
||||
except OSError:
|
||||
break
|
||||
parent = parent.parent
|
||||
|
||||
@@ -25,7 +25,7 @@ from sqlalchemy.orm import Session
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, probe, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.models import DownloadJob, MediaAsset, display_text
|
||||
|
||||
|
||||
def _title_from_filename(filename: str) -> str:
|
||||
@@ -99,7 +99,7 @@ def ingest_upload(
|
||||
source_kind="upload",
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
display_name=(display_name or "").strip() or None,
|
||||
display_name=display_text(display_name),
|
||||
status="done",
|
||||
progress=100,
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
|
||||
+15
-7
@@ -11,7 +11,6 @@ to be a person telling them nothing arrived.
|
||||
"""
|
||||
|
||||
import html
|
||||
import logging
|
||||
import re
|
||||
import smtplib
|
||||
import ssl
|
||||
@@ -21,15 +20,14 @@ from email.utils import formatdate
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import mailtext, sysconfig
|
||||
from app import logs, mailtext, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import User
|
||||
from app.notifications import create_notification
|
||||
from app.ratelimit import RateLimiter
|
||||
|
||||
log = logging.getLogger("siftlode.email")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# One admin notice an hour per reason. A relay that is down fails every send, and the inbox must
|
||||
# stay readable; the log keeps the full record either way.
|
||||
_alert_throttle = RateLimiter(max_events=1, window_seconds=3600)
|
||||
@@ -155,7 +153,15 @@ def _send(to: list[str], subject: str, body: str, reply_to: str | None = None) -
|
||||
return False
|
||||
c = _smtp()
|
||||
if not (c["host"] and c["user"] and c["password"]):
|
||||
log.info("SMTP not configured; skipping email %r to %s", subject, recipients)
|
||||
# The count, not the addresses: `redact_text` rewrites every one of them to `[email]`
|
||||
# before either sink sees the line, so the list only ever printed as noise. The subject
|
||||
# stays — the templates are static, and the one that interpolates (`admin_new_request`)
|
||||
# interpolates an address, which the same redactor covers.
|
||||
log.info(
|
||||
"SMTP not configured; skipping email %r to %d recipient(s)",
|
||||
subject,
|
||||
len(recipients), # log-pii-ok: how many, not who
|
||||
)
|
||||
_alert_admins("not_configured", subject, recipients)
|
||||
return False
|
||||
msg = EmailMessage()
|
||||
@@ -176,10 +182,12 @@ def _send(to: list[str], subject: str, body: str, reply_to: str | None = None) -
|
||||
s.starttls(context=ssl.create_default_context())
|
||||
s.login(c["user"], c["password"])
|
||||
s.send_message(msg)
|
||||
log.info("Sent email %r to %s", subject, recipients)
|
||||
# log-pii-ok: how many, not who
|
||||
log.info("Sent email %r to %d recipient(s)", subject, len(recipients))
|
||||
return True
|
||||
except Exception:
|
||||
log.exception("SMTP send failed (%r to %s)", subject, recipients)
|
||||
# log-pii-ok: how many, not who
|
||||
log.exception("SMTP send failed (%r to %d recipient(s))", subject, len(recipients))
|
||||
_alert_admins("send_failed", subject, recipients)
|
||||
return False
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ from datetime import date, datetime, timedelta, timezone
|
||||
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.collation import collated
|
||||
from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
@@ -30,7 +31,7 @@ from app.models import (
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.utils import iso, to_tsquery_str
|
||||
from app.utils import iso, like_escape, to_tsquery_str
|
||||
|
||||
|
||||
def _channel_url(channel_id: str, handle: str | None) -> str:
|
||||
@@ -272,8 +273,9 @@ def filtered_query(
|
||||
# `unaccent(channels.title) ILIKE '%q%'` straight into the per-video predicate defeated the
|
||||
# search_vector GIN — the planner detoasted every non-matching tsvector to test the `@@`
|
||||
# (~2.6s/search on the prod-sized set); channel_id.in_(...) keeps the GIN as a BitmapOr branch
|
||||
# (PERF-12). Empty match -> false() so the OR collapses to the text match alone.
|
||||
like = func.unaccent(f"%{q}%")
|
||||
# (PERF-12). Empty match -> false() so the OR collapses to the text match alone. The query
|
||||
# is a literal inside the pattern: `_` in `my_channel` is not a wildcard.
|
||||
like = func.unaccent(f"%{like_escape(q)}%")
|
||||
matching_channel_ids = (
|
||||
db.execute(select(Channel.id).where(func.unaccent(Channel.title).ilike(like)))
|
||||
.scalars()
|
||||
@@ -477,8 +479,10 @@ def feed_years(db: Session, user: User, **params) -> list[dict]:
|
||||
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
|
||||
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
||||
# (OFFSET would shift and duplicate/skip rows).
|
||||
def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) -> list[dict]:
|
||||
"""`lang` orders the title sorts in the reader's alphabet (`collation`); the other keys ignore it."""
|
||||
pub = {"expr": Video.published_at, "asc": False, "nullable": False, "kind": "dt"}
|
||||
title = collated(Video.title, lang)
|
||||
table: dict[str, list[dict]] = {
|
||||
"newest": [pub],
|
||||
"oldest": [{**pub, "asc": True}],
|
||||
@@ -490,10 +494,8 @@ def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
"duration_asc": [
|
||||
{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}
|
||||
],
|
||||
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
|
||||
"title_desc": [
|
||||
{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}
|
||||
],
|
||||
"title": [{"expr": title, "asc": True, "nullable": True, "kind": "str"}],
|
||||
"title_desc": [{"expr": title, "asc": False, "nullable": True, "kind": "str"}],
|
||||
"subscribers": [
|
||||
{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}
|
||||
],
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Bug reports built from debug sessions (E16): the files an issue carries, and where they go —
|
||||
downloaded as a zip, or (S4) sent to a Gitea issue tracker. The session itself — what is recorded
|
||||
and tagged — is the core module `app/debugsession.py`."""
|
||||
@@ -0,0 +1,79 @@
|
||||
"""What an issue from a debug session carries, as files — one definition for the zip export and the
|
||||
Gitea upload, so the two cannot drift apart.
|
||||
|
||||
* `issue.md` — the title and the body the user wrote and reviewed in the composer (rendered there,
|
||||
in their language; the server does not rewrite it);
|
||||
* `client.log` / `server.log` — the session's evidence (`debugsession.bundle`) minus the rows the
|
||||
user took out, one JSON object per line, browser rows and server rows apart. `.log`, not
|
||||
`.jsonl`: Gitea refuses that extension (measured, plans/e16 S0);
|
||||
* the screenshots, in the order they were taken.
|
||||
|
||||
Every row is already redacted twice (at capture, and again when stored), and nothing here adds a
|
||||
field the review list did not show.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, logsink
|
||||
from app.models import DebugScreenshot, DebugSession
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IssueFile:
|
||||
name: str
|
||||
data: bytes
|
||||
mime: str
|
||||
|
||||
|
||||
def title_of(session: DebugSession) -> str:
|
||||
# Not a sentence of ours: the composer always saves a title in the user's language, and a
|
||||
# session exported before it ever saved is named by its date alone, in no language at all.
|
||||
title = ((session.draft or {}).get("title") or "").strip()
|
||||
return title or f"{session.started_at:%Y-%m-%d %H:%M} · {session.id}"
|
||||
|
||||
|
||||
def body_of(session: DebugSession) -> str:
|
||||
return ((session.draft or {}).get("body") or "").strip()
|
||||
|
||||
|
||||
def _lines(rows) -> bytes:
|
||||
return "".join(
|
||||
json.dumps(logsink.serialize(r), ensure_ascii=False, default=str) + "\n" for r in rows
|
||||
).encode()
|
||||
|
||||
|
||||
def files(db: Session, session: DebugSession) -> list[IssueFile]:
|
||||
"""The session's issue as files: the text, the two logs, the screenshots."""
|
||||
rows, _ = debugsession.bundle(db, session, drop_excluded=True)
|
||||
text = f"# {title_of(session)}\n\n{body_of(session)}\n"
|
||||
out = [
|
||||
IssueFile("issue.md", text.encode(), "text/markdown"),
|
||||
IssueFile("client.log", _lines(r for r in rows if r.source == "web"), "text/plain"),
|
||||
IssueFile("server.log", _lines(r for r in rows if r.source != "web"), "text/plain"),
|
||||
]
|
||||
shots = db.scalars(
|
||||
select(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
.order_by(DebugScreenshot.id)
|
||||
)
|
||||
for n, shot in enumerate(shots, start=1):
|
||||
ext = debugsession.IMAGE_TYPES[shot.mime]
|
||||
name = f"{debugsession.SCREENSHOT_PREFIX}{n}.{ext}"
|
||||
out.append(IssueFile(name, shot.data, shot.mime))
|
||||
return out
|
||||
|
||||
|
||||
def zipped(items: list[IssueFile]) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
|
||||
for item in items:
|
||||
# The images are compressed already: deflating them again costs time and wins nothing.
|
||||
method = zipfile.ZIP_STORED if item.mime.startswith("image/") else zipfile.ZIP_DEFLATED
|
||||
z.writestr(item.name, item.data, compress_type=method)
|
||||
return buf.getvalue()
|
||||
@@ -0,0 +1,288 @@
|
||||
"""The Gitea issue tracker a debug session's issue goes to (E16 S4) — measured against Gitea 1.27.1
|
||||
before a line of this was written (plans/e16-debug-issue.md, S0).
|
||||
|
||||
Configured by an admin (sysconfig `issues_gitea_url` / `_repo` / `_token`, the token encrypted at
|
||||
rest). The repository must be PRIVATE: an issue carries its reporter's log — page titles, filter
|
||||
values, at Teljes (TRACE) what they typed into search — and a public repository would publish it. That is
|
||||
checked on every send, not only in the admin's "test connection", because a repository can be made
|
||||
public after the fact.
|
||||
|
||||
What an issue is made of comes from `bundle.files` (the same files the zip export carries). Sending
|
||||
is three kinds of call, measured:
|
||||
* `POST /repos/{repo}/issues` — the title and the body the user wrote;
|
||||
* `POST /repos/{repo}/issues/{n}/assets?name=…` — ONE file per request (field `attachment`);
|
||||
* `PATCH /repos/{repo}/issues/{n}` — the body again, with the screenshots shown inline once their
|
||||
links exist.
|
||||
A private repository's attachment answers 404 to a browser not signed in to Gitea, which is why the
|
||||
composer's own preview shows our copies, never these links.
|
||||
|
||||
Delivery keeps its progress in the session's draft (`delivery`: the issue number, the files still to
|
||||
upload, the links of those uploaded, whether the body shows the screenshots yet), so a failure
|
||||
half-way keeps the issue that exists and a retry does only what is missing — never a second issue.
|
||||
Every step is checkpointed (the caller's `db.commit`), so what did happen is kept whatever fails
|
||||
next — and the first checkpoint comes before the first call, with the tracker's settings read ABOVE
|
||||
it, so neither a row lock nor a read snapshot nor a pooled connection is held across the calls. The
|
||||
caller claims the session for the send first (`debugsession.claim_send`) and passes the claim's
|
||||
token in, which is what keeps a second send, an autosave or a screenshot out of the way; the claim
|
||||
is renewed by every progress report, and a report whose claim is gone raises rather than writing
|
||||
over the send that took it. `debugsession.not_uploaded` / `images_pending` read that progress.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, logs, sysconfig
|
||||
from app.issues import bundle
|
||||
from app.models import DebugSession
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
TIMEOUT_SECONDS = 20.0
|
||||
|
||||
Reason = Literal[
|
||||
"not_configured", # url, repository or token missing
|
||||
"unreachable", # no answer, or a 5xx
|
||||
"bad_token", # 401/403: the token is wrong or lacks write:issue / read:repository
|
||||
"repo_not_found", # 404 on the repository
|
||||
"public_repo", # the repository is public — an issue would publish the reporter's log
|
||||
"issues_disabled", # the repository has its issue tracker switched off
|
||||
"rejected", # any other 4xx (a file type the instance does not allow, …), or an answer that is
|
||||
# not the API's (a redirect — an http:// address behind https —, a page that is not JSON)
|
||||
]
|
||||
|
||||
|
||||
class GiteaError(Exception):
|
||||
def __init__(self, reason: Reason) -> None:
|
||||
super().__init__(reason)
|
||||
self.reason: Reason = reason
|
||||
|
||||
|
||||
def _settings(db: Session) -> tuple[str, str, str]:
|
||||
return (
|
||||
sysconfig.get_str(db, "issues_gitea_url").strip().rstrip("/"),
|
||||
sysconfig.get_str(db, "issues_gitea_repo").strip().strip("/"),
|
||||
sysconfig.get_str(db, "issues_gitea_token").strip(),
|
||||
)
|
||||
|
||||
|
||||
def target(db: Session) -> dict | None:
|
||||
"""Where a report goes, for the composer to say so — the repository — or None when no tracker is
|
||||
configured (the report can then only be downloaded)."""
|
||||
url, repo, token = _settings(db)
|
||||
return {"repo": repo} if url and repo and token else None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _client(settings: tuple[str, str, str]) -> Iterator[tuple[httpx.Client, str]]:
|
||||
# Takes the settings already read, not the session: reading them HERE opened a transaction one
|
||||
# line after `deliver` had ended one, and held it across the calls below (deep-review round 4).
|
||||
url, repo, token = settings
|
||||
if not (url and repo and token):
|
||||
raise GiteaError("not_configured")
|
||||
with httpx.Client(
|
||||
base_url=f"{url}/api/v1",
|
||||
headers={"Authorization": f"token {token}", "Accept": "application/json"},
|
||||
timeout=TIMEOUT_SECONDS,
|
||||
) as client:
|
||||
yield client, repo
|
||||
|
||||
|
||||
def _call(client: httpx.Client, method: str, path: str, **kwargs) -> httpx.Response:
|
||||
try:
|
||||
r = client.request(method, path, **kwargs)
|
||||
except httpx.HTTPError as exc:
|
||||
log.warning(
|
||||
"the issue tracker did not answer",
|
||||
extra=logs.event("issues.gitea.unreachable", path=path, error=type(exc).__name__),
|
||||
)
|
||||
raise GiteaError("unreachable") from None
|
||||
if r.status_code < 300:
|
||||
return r
|
||||
# The status only: Gitea's message can quote back what was sent — the user's own title.
|
||||
log.warning(
|
||||
"the issue tracker refused a call",
|
||||
extra=logs.event("issues.gitea.refused", path=path, status=r.status_code),
|
||||
)
|
||||
if r.status_code in (401, 403):
|
||||
raise GiteaError("bad_token")
|
||||
if r.status_code == 404:
|
||||
raise GiteaError("repo_not_found")
|
||||
if r.status_code >= 500:
|
||||
raise GiteaError("unreachable")
|
||||
raise GiteaError("rejected")
|
||||
|
||||
|
||||
def _json(r: httpx.Response, *keys: str) -> dict[str, Any]:
|
||||
"""The answer as the API's JSON, holding `keys` — anything else is refused, not a crash, so a
|
||||
step after the issue was created cannot take down the request that holds its record.
|
||||
|
||||
The CREATE call is the one place this cannot keep that record: its answer IS the record. An
|
||||
answer it cannot read there leaves the session unsent, and a retry opens another issue — the
|
||||
price of not being able to tell a refused create from an unreadable one."""
|
||||
try:
|
||||
data = r.json()
|
||||
# silent-ok: not JSON is one of the answers refused (and logged) just below.
|
||||
except ValueError:
|
||||
data = None
|
||||
if not isinstance(data, dict) or any(k not in data for k in keys):
|
||||
log.warning(
|
||||
"the issue tracker answered something that is not its API",
|
||||
extra=logs.event("issues.gitea.unexpected", status=r.status_code),
|
||||
)
|
||||
raise GiteaError("rejected")
|
||||
return data
|
||||
|
||||
|
||||
def _private_repo(client: httpx.Client, repo: str) -> dict:
|
||||
data = _json(_call(client, "GET", f"/repos/{repo}"))
|
||||
if not data.get("private"):
|
||||
raise GiteaError("public_repo")
|
||||
if not data.get("has_issues"):
|
||||
raise GiteaError("issues_disabled")
|
||||
return data
|
||||
|
||||
|
||||
def check(db: Session) -> dict:
|
||||
"""The admin's "test connection": the repository answers, is private and takes issues."""
|
||||
with _client(_settings(db)) as (client, repo):
|
||||
data = _private_repo(client, repo)
|
||||
return {"repo": data.get("full_name") or repo, "url": data.get("html_url")}
|
||||
|
||||
|
||||
def preview(db: Session, text: str) -> str:
|
||||
"""The body as Gitea itself will render it (its markdown flavour, its sanitiser).
|
||||
|
||||
**No `Context`, deliberately — a repository context here does not fail the call, it kills the
|
||||
tracker.** With one, Gitea's post-processor turns every hex-shaped word into a commit link, and
|
||||
on the API's render path the commit checker it asks is nil: the whole Gitea process segfaults
|
||||
and restarts, taking the git server (and, here, the container registry) down with it. Every
|
||||
report carries hex — a tab id, a request id — so this was not an edge case but the only case.
|
||||
Measured on throwaway instances (2026-09-20): `{"Mode": "gfm", "Context": "<repo>"}` with
|
||||
`5645b1c0ae01` in the text crashes 1.26.4 and 1.27.1 and is fixed in 1.27.3 — but a self-hoster
|
||||
runs whatever version they run, so the payload stays safe here rather than relying on theirs.
|
||||
Dropping `Context` is not a downgrade of the preview: measured against the same body, the HTML
|
||||
keeps its tables, its task lists and its highlighted code blocks. It only stops resolving
|
||||
repo-relative links (a hash, a `#123`), which the issue page itself still resolves.
|
||||
"""
|
||||
with _client(_settings(db)) as (client, _repo):
|
||||
r = _call(client, "POST", "/markdown", json={"Text": text, "Mode": "gfm"})
|
||||
return r.text
|
||||
|
||||
|
||||
def deliver(
|
||||
db: Session,
|
||||
session: DebugSession,
|
||||
*,
|
||||
token: str,
|
||||
checkpoint: Callable[[], None],
|
||||
before_create: Callable[[], None] | None = None,
|
||||
on_created: Callable[[], None] | None = None,
|
||||
) -> str:
|
||||
"""Send the session's issue, or finish sending it; returns its URL.
|
||||
|
||||
`token` is the caller's claim on this session (`debugsession.claim_send`). Every progress report
|
||||
renews it and checks the row still holds it, so a send that outlived its lease stops here rather
|
||||
than writing over the one that took over (`debugsession.SendClaimLost`).
|
||||
|
||||
`checkpoint` persists what has happened so far and is called after every step (the caller's
|
||||
commit — this module does not decide when a transaction ends). `before_create` runs right before
|
||||
the tracker is asked to open a NEW issue, after the checks that can refuse it without asking,
|
||||
and may raise to stop it; `on_created` runs right after one was opened, in time to be committed
|
||||
by the SAME checkpoint that first writes `issue_url` — anything the caller must not be able to
|
||||
lose separately from the link belongs there.
|
||||
"""
|
||||
state = debugsession.delivery(session)
|
||||
files = bundle.files(db, session)
|
||||
attachments = [f for f in files if f.name != "issue.md"]
|
||||
settings = _settings(db)
|
||||
# Reading the bundle and the tracker's settings opened a transaction; end it before the first
|
||||
# call, so nothing holds a snapshot (and an idle connection) across minutes of tracker I/O.
|
||||
# The settings are read HERE and handed to `_client`, not read inside it: reading them there
|
||||
# was a SELECT one line after this commit, which reopened the transaction and held it — idle,
|
||||
# with its snapshot pinned — across the repository check and the create (deep-review round 4
|
||||
# measured it in pg_stat_activity). A comment saying otherwise sat right here.
|
||||
checkpoint()
|
||||
with _client(settings) as (client, repo):
|
||||
_private_repo(client, repo)
|
||||
if "number" not in state:
|
||||
if before_create is not None:
|
||||
before_create()
|
||||
created = _json(
|
||||
_call(
|
||||
client,
|
||||
"POST",
|
||||
f"/repos/{repo}/issues",
|
||||
json={"title": bundle.title_of(session), "body": bundle.body_of(session)},
|
||||
),
|
||||
"number",
|
||||
"html_url",
|
||||
)
|
||||
# Built ON the state read at the start so nothing already in it is dropped. The claim
|
||||
# is NOT among those things, whatever this comment used to say: `set_delivery` strips
|
||||
# any `sending` passed to it and stamps the row's own from `token` (round 4 — removing
|
||||
# the spread left the whole suite green, because the claim never travelled here).
|
||||
state = {
|
||||
**state,
|
||||
"number": created["number"],
|
||||
"pending": [f.name for f in attachments],
|
||||
"links": {},
|
||||
}
|
||||
session.issue_url = created["html_url"]
|
||||
try:
|
||||
debugsession.set_delivery(db, session, state, token)
|
||||
except debugsession.SendClaimLost:
|
||||
# The one step where losing the claim costs something irreversible: the issue is
|
||||
# already open on the tracker and this is what would have recorded its link. We
|
||||
# still must not write — another send owns the row and is opening its own issue —
|
||||
# so say which issue was left behind, because nothing else will.
|
||||
# noqa TRY400: the traceback is one SendClaimLost carrying the session id and
|
||||
# nothing else; the issue number below is the whole news.
|
||||
log.error( # noqa: TRY400
|
||||
"an issue was created by a send that had already lost its claim",
|
||||
extra=logs.event(
|
||||
"issues.gitea.orphaned",
|
||||
number=created["number"],
|
||||
session_id=session.id,
|
||||
),
|
||||
)
|
||||
raise
|
||||
if on_created is not None:
|
||||
on_created()
|
||||
checkpoint()
|
||||
log.info(
|
||||
"issue created",
|
||||
extra=logs.event("issues.gitea.created", number=created["number"]),
|
||||
)
|
||||
number = state["number"]
|
||||
for item in attachments:
|
||||
if item.name not in state["pending"]:
|
||||
continue
|
||||
asset = _json(
|
||||
_call(
|
||||
client,
|
||||
"POST",
|
||||
f"/repos/{repo}/issues/{number}/assets",
|
||||
params={"name": item.name},
|
||||
files={"attachment": (item.name, item.data, item.mime)},
|
||||
),
|
||||
"browser_download_url",
|
||||
)
|
||||
state = {
|
||||
**state,
|
||||
"pending": [n for n in state["pending"] if n != item.name],
|
||||
"links": {**state["links"], item.name: asset["browser_download_url"]},
|
||||
}
|
||||
debugsession.set_delivery(db, session, state, token)
|
||||
checkpoint()
|
||||
shots = [n for n in state["links"] if n.startswith(debugsession.SCREENSHOT_PREFIX)]
|
||||
if shots and not state.get("images_shown"):
|
||||
images = "\n".join(f"" for n in sorted(shots))
|
||||
body = f"{bundle.body_of(session)}\n\n---\n\n{images}"
|
||||
_call(client, "PATCH", f"/repos/{repo}/issues/{number}", json={"body": body})
|
||||
debugsession.set_delivery(db, session, {**state, "images_shown": True}, token)
|
||||
checkpoint()
|
||||
return session.issue_url or ""
|
||||
@@ -0,0 +1,838 @@
|
||||
"""Structured logging for every Siftlode process — the ONE place logging is configured.
|
||||
|
||||
Before this module a single deployment printed FOUR different log formats: uvicorn's
|
||||
`log_config.json` for the API, a hand-rolled StreamHandler in `main.py` for everything that did not
|
||||
start under uvicorn (tests, scripts), `logging.basicConfig()` in the worker and in the download
|
||||
child (no timestamp at all), and alembic's `fileConfig`. None of them carried a correlation id, and
|
||||
uvicorn's access log printed the query string of every request (`?account=1`, `?source=feed`) — so
|
||||
the logs were both unjoinable and leaky. `configure_logging()` replaces all four.
|
||||
|
||||
Event shape — the OpenTelemetry log data model, without the SDK; one JSON object per line:
|
||||
ts · level · sev · source · logger · event · msg · attrs
|
||||
plus whatever correlation the context carries (`request_id`, `user_id`, `debug_session_id`,
|
||||
`job_id`) and the app version. That shape is what E22.6's admin console filters on and what a Gitea
|
||||
issue attachment will carry, so it is deliberately flat and stable.
|
||||
|
||||
Call sites stay ordinary stdlib logging:
|
||||
|
||||
log = logs.get_logger(__name__) # app/downloads/pipeline.py -> siftlode.downloads.pipeline
|
||||
log.info("job finished", extra=logs.event("download.job.done", job_id=7, bytes=n))
|
||||
|
||||
`event()` is optional — a bare `log.info("…")` still comes out well-formed — but the dotted code is
|
||||
what makes a line queryable later, so add one wherever the line matters.
|
||||
|
||||
REDACTION IS APPLIED AT THE HANDLER, not at the call site: every message, every attr value and
|
||||
every traceback passes `redact()` on the way out. That is the whole point — a secret cannot be
|
||||
leaked by forgetting to redact it at one of ~125 call sites, and a new call site is safe by
|
||||
default. Never logged, at any level: passwords, tokens (session, reset/verify, setup, share,
|
||||
OAuth), cookie blobs, secret config values, and email addresses.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import logging.config
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from app.config import settings
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
|
||||
# One level below DEBUG for the "Teljes / TRACE" diagnostics level (search strings, redacted request
|
||||
# bodies). Registered on import so `logging.getLogger(...).log(logs.TRACE, …)` and a `LOG_LEVEL=TRACE`
|
||||
# both work in every process, including one that never calls configure_logging (a test).
|
||||
TRACE = 5
|
||||
logging.addLevelName(TRACE, "TRACE")
|
||||
|
||||
# OpenTelemetry severity numbers for our level names. Kept beside the level because the number is
|
||||
# what a log backend sorts and filters on; the name is for humans reading a terminal.
|
||||
SEVERITY: dict[str, int] = {
|
||||
"TRACE": 1,
|
||||
"DEBUG": 5,
|
||||
"INFO": 9,
|
||||
"WARNING": 13,
|
||||
"ERROR": 17,
|
||||
"CRITICAL": 21,
|
||||
}
|
||||
|
||||
# Which process this is. Set by configure_logging; every line carries it so one Loki/DB query can
|
||||
# separate the API's view of a job from the worker's.
|
||||
_source = "api"
|
||||
|
||||
|
||||
# --- correlation context -------------------------------------------------------------------
|
||||
# contextvars, like quota's actor attribution (app/quota.py): set at an entry point, read by the
|
||||
# formatter, never threaded through call signatures. A child task copies the context at creation,
|
||||
# so anything set before `await call_next(...)` is visible to the whole request.
|
||||
_request_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"log_request_id", default=None
|
||||
)
|
||||
_user_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_user_id", default=None)
|
||||
_debug_session: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"log_debug_session", default=None
|
||||
)
|
||||
_job_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_job_id", default=None)
|
||||
|
||||
_CONTEXT_VARS = {
|
||||
"request_id": _request_id,
|
||||
"user_id": _user_id,
|
||||
"debug_session_id": _debug_session,
|
||||
"job_id": _job_id,
|
||||
}
|
||||
|
||||
|
||||
class _RequestScope:
|
||||
"""What the auth dependency learns about a request, for every line logged after it.
|
||||
|
||||
A MUTABLE object behind a contextvar, not plain contextvars, because of how a request actually
|
||||
runs: FastAPI resolves a sync dependency (`auth.current_user`) in a threadpool on a COPY of the
|
||||
context, so a `ContextVar.set()` there is gone by the time the handler runs — measured: that is
|
||||
how the account never reached a handler's own lines. `log_requests` installs one scope per
|
||||
request; every copied context shares the same object, so what the dependency writes is seen by
|
||||
the handler and by the request line itself."""
|
||||
|
||||
__slots__ = ("user_id", "level", "debug_session_id")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.user_id: int | None = None
|
||||
# The user's consented diagnostics level as a logging number; None at the base level.
|
||||
self.level: int | None = None
|
||||
# The user's own open debug session this request named (E16), else None.
|
||||
self.debug_session_id: str | None = None
|
||||
|
||||
|
||||
_scope: contextvars.ContextVar[_RequestScope | None] = contextvars.ContextVar(
|
||||
"log_request_scope", default=None
|
||||
)
|
||||
|
||||
|
||||
def set_user(
|
||||
user_id: int | None,
|
||||
level: str = "INFO",
|
||||
until: datetime | None = None,
|
||||
debug_session: str | None = None,
|
||||
) -> None:
|
||||
"""Attribute the rest of this request's log lines to `user_id`, and log them at the user's
|
||||
consented diagnostics `level` until `until` (E22 S5; "INFO" = no elevation). Called from the
|
||||
auth dependency — the one place the session is actually resolved into an account — rather than
|
||||
from the request middleware, which runs OUTSIDE the session middleware.
|
||||
|
||||
`debug_session` tags every line with the user's open debug session (E16). The caller has
|
||||
already checked it is theirs and open; a session's level arrives here as `level`/`until`.
|
||||
|
||||
The root logger's floor is tracked per elevation: a request without the session's header (an
|
||||
`<img>`, a `<video>` — the browser sends those itself) carries only the user's own level, and
|
||||
must not lower the floor a concurrent request of the session still needs."""
|
||||
_user_id.set(user_id)
|
||||
scope = _scope.get()
|
||||
if scope is None: # outside log_requests (a script, a unit test): this context only
|
||||
scope = _RequestScope()
|
||||
_scope.set(scope)
|
||||
elevated = level != "INFO" and until is not None and until > datetime.now(timezone.utc)
|
||||
scope.user_id = user_id
|
||||
scope.debug_session_id = debug_session
|
||||
scope.level = resolve_level(level) if elevated else None
|
||||
if user_id is not None:
|
||||
key = user_id if debug_session is None else (user_id, debug_session)
|
||||
_track_elevation(key, scope.level, until)
|
||||
|
||||
|
||||
def set_job(job_id: int | None) -> None:
|
||||
"""Attribute this process's log lines to a download job — for the download CHILD, whose whole
|
||||
life is one job (the worker uses `bind()` per job instead, since it juggles several)."""
|
||||
_job_id.set(job_id)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def bind(**fields: Any):
|
||||
"""Bind correlation fields for the duration of a block (a worker job, a scheduler run):
|
||||
|
||||
with logs.bind(job_id=job.id):
|
||||
...
|
||||
|
||||
Unknown field names are ignored rather than raising — a caller binding a field this module does
|
||||
not know about is a typo, not a reason to take down a download."""
|
||||
tokens = [(_CONTEXT_VARS[k].set(v)) for k, v in fields.items() if k in _CONTEXT_VARS]
|
||||
keys = [k for k in fields if k in _CONTEXT_VARS]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for key, token in zip(keys, tokens):
|
||||
_CONTEXT_VARS[key].reset(token)
|
||||
|
||||
|
||||
def event(code: str, /, **attrs: Any) -> dict[str, Any]:
|
||||
"""`extra=` payload for a log call: a stable dotted event code plus structured attributes.
|
||||
|
||||
Returns a plain dict so it composes with stdlib logging and costs nothing when the level is off
|
||||
(the dict is built either way; keep the attrs cheap — no DB reads inside the call).
|
||||
|
||||
`code` is positional-only so that no attribute name is reserved: `event("x", code=...)` means
|
||||
the attribute, not this parameter. Without the `/` it was a TypeError, and `code` is exactly
|
||||
the word a caller reaches for when the thing that failed has one of its own."""
|
||||
return {"event": code, "attrs": attrs}
|
||||
|
||||
|
||||
# --- redaction -----------------------------------------------------------------------------
|
||||
# Substring matching, not exact keys: the same secret travels as `password`, `current_password`,
|
||||
# `new_password`, `smtp_password`, `plex_server_token`, `access_token`, … Matching on the PART is
|
||||
# what makes a key we have never seen safe by default.
|
||||
_SECRET_KEY_PARTS = (
|
||||
"password",
|
||||
"passphrase",
|
||||
"token",
|
||||
"secret",
|
||||
"cookie",
|
||||
"authorization",
|
||||
"auth_header",
|
||||
"ciphertext",
|
||||
"private_key",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"credential",
|
||||
"session_key",
|
||||
)
|
||||
|
||||
# Query/fragment parameters that carry a credential in a URL we log (OAuth `code`/`state`, our own
|
||||
# `?token=` setup and share links, `#verify=` / `#reset=` fragments).
|
||||
_SECRET_PARAM_NAMES = (
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"id_token",
|
||||
"token",
|
||||
"code",
|
||||
"state",
|
||||
"key",
|
||||
"secret",
|
||||
"password",
|
||||
"pwd",
|
||||
"sig",
|
||||
"signature",
|
||||
"verify",
|
||||
"reset",
|
||||
"setup",
|
||||
)
|
||||
_SECRET_PARAM_RE = re.compile(
|
||||
r"\b((?:" + "|".join(_SECRET_PARAM_NAMES) + r")=)([^&\s\"']+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Deliberately loose: over-matching redacts a non-address that looks like one (harmless), while
|
||||
# under-matching puts a real address in a log we attach to an issue (not harmless).
|
||||
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
REDACTED = "[redacted]"
|
||||
# `extra={PERSIST: False}` keeps a line out of the `log_events` table while stdout still gets it —
|
||||
# how `log_requests` caps anonymous traffic per client (app/logsink.py reads it).
|
||||
PERSIST = "persist"
|
||||
REDACTED_EMAIL = "[email]"
|
||||
|
||||
|
||||
def redact_text(text: str) -> str:
|
||||
"""Scrub a free-text string: credential-bearing URL params and email addresses."""
|
||||
scrubbed = _SECRET_PARAM_RE.sub(lambda m: f"{m.group(1)}{REDACTED}", text)
|
||||
return _EMAIL_RE.sub(REDACTED_EMAIL, scrubbed)
|
||||
|
||||
|
||||
def redact_query(query: str) -> str:
|
||||
"""Scrub a raw query string. DECODED first: the text rules match what was typed, and a browser
|
||||
sends it percent-encoded — `someone%40example.com` is an address `_EMAIL_RE` cannot see. Each
|
||||
parameter also goes through the key rules, so `?new_password=` goes as `redact` would drop it."""
|
||||
pairs = parse_qsl(query, keep_blank_values=True)
|
||||
return "&".join(f"{key}={_redact_param(key, value)}" for key, value in pairs)
|
||||
|
||||
|
||||
def _redact_param(key: str, value: str) -> str:
|
||||
"""One decoded `key=value` pair. The parameter-NAME rule is applied to the key here rather than
|
||||
left to `_SECRET_PARAM_RE` over the rebuilt pair: that pattern was written for the raw,
|
||||
percent-encoded query, where a value holds no literal space, `&` or quote, and its value class
|
||||
stops at the first one. On decoded text `?state=a%20b` would keep its tail and `?code=%20abc`
|
||||
would not match at all — the whole credential straight into the line."""
|
||||
if key.lower() in _SECRET_PARAM_NAMES:
|
||||
return REDACTED
|
||||
return str(redact(value, key=key))
|
||||
|
||||
|
||||
def redact(value: Any, *, key: str = "") -> Any:
|
||||
"""Recursively scrub a structure destined for the log.
|
||||
|
||||
A value whose KEY names a secret is dropped whole (we never log a prefix of a token — a prefix
|
||||
is still a leak and still useless). Everything else is scrubbed as text. Depth is bounded by
|
||||
the structures we actually log (attrs dicts), and non-JSON types fall through to `str`."""
|
||||
lowered = key.lower()
|
||||
if key and any(part in lowered for part in _SECRET_KEY_PARTS):
|
||||
return REDACTED
|
||||
if isinstance(value, dict):
|
||||
return {k: redact(v, key=str(k)) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [redact(v, key=key) for v in value]
|
||||
if isinstance(value, str):
|
||||
return redact_text(value)
|
||||
return value
|
||||
|
||||
|
||||
# --- formatters ----------------------------------------------------------------------------
|
||||
_SCRUBBED_STACK = "scrubbed_stack"
|
||||
|
||||
|
||||
def scrubbed_stack(exc: BaseException, scrub: Callable[[BaseException], str]) -> dict[str, str]:
|
||||
"""`extra=` fragment: `exc`'s traceback, rendered where an `exc_info` stack goes, with the
|
||||
message of every exception in it put through `scrub`.
|
||||
|
||||
For a failure whose message can name someone's data (a download's URL, a file path, a video
|
||||
id). The frames name code and are kept whole; the text at the foot of each link is the data.
|
||||
A plain `exc_info` renders the chain too — the `__cause__`/`__context__` a DB error raised
|
||||
inside `except DownloadError` carries — so scrubbing only the line's own message is not
|
||||
enough. Merge it into the call's `extra`: `{**event(...), **scrubbed_stack(...)}`."""
|
||||
links: list[BaseException] = []
|
||||
link: BaseException | None = exc
|
||||
while link is not None and all(link is not seen for seen in links):
|
||||
links.append(link)
|
||||
link = link.__cause__ or (None if link.__suppress_context__ else link.__context__)
|
||||
parts: list[str] = []
|
||||
for i in range(len(links) - 1, -1, -1): # the oldest first, as Python prints a chain
|
||||
link = links[i]
|
||||
kind = type(link)
|
||||
name = kind.__qualname__
|
||||
if kind.__module__ not in ("builtins", "__main__"):
|
||||
name = f"{kind.__module__}.{name}"
|
||||
try:
|
||||
message = scrub(link)
|
||||
# silent-ok: the message is what could not be rendered, and the placeholder says so — the
|
||||
# caller is in the middle of reporting one failure and must not get a second from here.
|
||||
except Exception: # noqa: BLE001 — a `__str__` that raises, of any shape
|
||||
message = "<unprintable>"
|
||||
frames = "".join(traceback.format_tb(link.__traceback__))
|
||||
head = f"Traceback (most recent call last):\n{frames}" if frames else ""
|
||||
parts.append(f"{head}{name}: {message}" if message else f"{head}{name}")
|
||||
if i:
|
||||
parts.append(
|
||||
"\nThe above exception was the direct cause of the following exception:\n"
|
||||
if links[i - 1].__cause__ is link
|
||||
else "\nDuring handling of the above exception, another exception occurred:\n"
|
||||
)
|
||||
return {_SCRUBBED_STACK: "\n".join(parts)}
|
||||
|
||||
|
||||
# Attributes every LogRecord carries; anything else a caller passed via `extra=` is a structured
|
||||
# attribute and is merged into `attrs` (so `extra={"job_id": 7}` works as well as `event(...)`).
|
||||
_RECORD_STANDARD = frozenset(
|
||||
vars(logging.LogRecord("", 0, "", 0, "", None, None)).keys()
|
||||
| {"message", "asctime", "taskName", "event", "attrs", PERSIST, "elevated", _SCRUBBED_STACK}
|
||||
)
|
||||
|
||||
|
||||
class ContextFilter(logging.Filter):
|
||||
"""Copies the correlation contextvars onto the record. A Filter (not the formatter) so the
|
||||
values are captured at emit time, before a QueueHandler can move the record to another
|
||||
thread — the DB sink in S3 depends on that."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
for name, var in _CONTEXT_VARS.items():
|
||||
value = var.get()
|
||||
if value is not None and not hasattr(record, name):
|
||||
setattr(record, name, value)
|
||||
scope = _scope.get()
|
||||
if scope is not None and scope.user_id is not None and not hasattr(record, "user_id"):
|
||||
record.user_id = scope.user_id
|
||||
if (
|
||||
scope is not None
|
||||
and scope.debug_session_id is not None
|
||||
and not hasattr(record, "debug_session_id")
|
||||
):
|
||||
record.debug_session_id = scope.debug_session_id
|
||||
return True
|
||||
|
||||
|
||||
class LevelGate(logging.Filter):
|
||||
"""The configured level, enforced at the HANDLER, with one exception: the lines of a request
|
||||
whose user raised their diagnostics level (E22 S5).
|
||||
|
||||
The root logger's level cannot do this alone: it is one number for the whole process, and the
|
||||
point is to log ONE user's requests in detail while everyone else's stay at the base level. So
|
||||
the root is lowered to the lowest level any active elevation needs (`_track_elevation`), and
|
||||
this gate drops what only an elevation let through unless the line belongs to that elevated
|
||||
request. It also marks such a line `elevated`, which is what makes the `log_events` sink keep
|
||||
every line of that request, not just its warnings."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
scope = _scope.get()
|
||||
if scope is not None and scope.level is not None:
|
||||
record.elevated = True
|
||||
return record.levelno >= min(scope.level, _base_level)
|
||||
return record.levelno >= _base_level
|
||||
|
||||
|
||||
def _record_attrs(record: logging.LogRecord) -> dict[str, Any]:
|
||||
extra = {k: v for k, v in vars(record).items() if k not in _RECORD_STANDARD}
|
||||
attrs = dict(getattr(record, "attrs", None) or {})
|
||||
attrs.update({k: v for k, v in extra.items() if k not in _CONTEXT_VARS})
|
||||
return attrs
|
||||
|
||||
|
||||
def _safe_message(record: logging.LogRecord) -> str:
|
||||
"""`record.getMessage()` raises if the args don't match the format string. A broken log call
|
||||
must not take down the caller, so the failure is reported IN the line instead."""
|
||||
try:
|
||||
return record.getMessage()
|
||||
# silent-ok: the failure is reported IN the returned line. Logging it would re-enter this same
|
||||
# formatter with a record built out of the one that just failed.
|
||||
except Exception as exc: # noqa: BLE001 — any formatting error, reported not raised
|
||||
return f"{record.msg!r} (log formatting failed: {exc})"
|
||||
|
||||
|
||||
def record_fields(record: logging.LogRecord) -> dict[str, Any]:
|
||||
"""The event as structured fields, REDACTED — the one definition both the stdout line and the
|
||||
`log_events` row are built from, so the two sinks can never disagree about what is safe to keep.
|
||||
Absent optional fields are left out; `ts` is an aware datetime."""
|
||||
fields: dict[str, Any] = {
|
||||
"ts": datetime.fromtimestamp(record.created, timezone.utc),
|
||||
"level": record.levelname,
|
||||
"sev": SEVERITY.get(record.levelname, 9),
|
||||
"source": _source,
|
||||
"logger": record.name,
|
||||
"msg": redact_text(_safe_message(record)),
|
||||
}
|
||||
code = getattr(record, "event", None)
|
||||
if code:
|
||||
fields["event"] = code
|
||||
for name in _CONTEXT_VARS:
|
||||
value = getattr(record, name, None)
|
||||
if value is not None:
|
||||
fields[name] = value
|
||||
attrs = _record_attrs(record)
|
||||
if attrs:
|
||||
fields["attrs"] = redact(attrs)
|
||||
if record.exc_info:
|
||||
fields["exc"] = redact_text(logging.Formatter().formatException(record.exc_info))
|
||||
elif getattr(record, _SCRUBBED_STACK, None):
|
||||
fields["exc"] = redact_text(getattr(record, _SCRUBBED_STACK))
|
||||
elif record.exc_text:
|
||||
# A QueueHandler-style hand-off formats the traceback into exc_text and drops exc_info.
|
||||
fields["exc"] = redact_text(record.exc_text)
|
||||
fields["version"] = settings.app_version
|
||||
return fields
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line — the shape Loki and the `log_events` table both ingest."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload = record_fields(record)
|
||||
payload["ts"] = payload["ts"].isoformat(timespec="milliseconds")
|
||||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
class TextFormatter(logging.Formatter):
|
||||
"""Human-readable single line for a terminal (`LOG_FORMAT=text`): the same content, ordered for
|
||||
reading. Redaction applies here too — a dev log is still a log someone pastes into an issue."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
stamp = datetime.fromtimestamp(record.created, timezone.utc).strftime("%H:%M:%S.%f")[:-3]
|
||||
head = f"{stamp} {record.levelname:<5} [{record.name}]"
|
||||
rid = getattr(record, "request_id", None)
|
||||
if rid:
|
||||
head += f" {rid}"
|
||||
code = getattr(record, "event", None)
|
||||
line = f"{head} {code + ' — ' if code else ''}{redact_text(_safe_message(record))}"
|
||||
attrs = _record_attrs(record)
|
||||
if attrs:
|
||||
line += f" {json.dumps(redact(attrs), ensure_ascii=False, default=str)}"
|
||||
if record.exc_info:
|
||||
line += "\n" + redact_text(self.formatException(record.exc_info))
|
||||
elif getattr(record, _SCRUBBED_STACK, None):
|
||||
line += "\n" + redact_text(getattr(record, _SCRUBBED_STACK))
|
||||
return line
|
||||
|
||||
|
||||
# --- configuration -------------------------------------------------------------------------
|
||||
# Third-party loggers that are noisy at our own level. Pinned explicitly so the diagnostics knob can
|
||||
# lower the ROOT level to DEBUG/TRACE without drowning the output in SQL statements and HTTP chatter
|
||||
# — raising THESE is a deliberate, separate act (E22.6's console).
|
||||
_THIRD_PARTY_LEVELS = {
|
||||
"sqlalchemy": "WARNING",
|
||||
"sqlalchemy.engine": "WARNING",
|
||||
"apscheduler": "WARNING",
|
||||
"urllib3": "WARNING",
|
||||
"httpx": "WARNING",
|
||||
"httpcore": "WARNING",
|
||||
"asyncio": "WARNING",
|
||||
"PIL": "WARNING",
|
||||
"alembic": "INFO",
|
||||
# uvicorn's own startup/shutdown/WebSocket lines are worth keeping; its ACCESS log is not —
|
||||
# it is one line per request with the query string in it, replaced by `log_requests` below.
|
||||
"uvicorn": "INFO",
|
||||
"uvicorn.error": "INFO",
|
||||
"uvicorn.access": "CRITICAL",
|
||||
}
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""The logger for one module: `logs.get_logger(__name__)` in `app/plex/sync.py` is
|
||||
`siftlode.plex.sync`.
|
||||
|
||||
The names were literals before, and 25 of them covered 34 modules — six different Plex modules
|
||||
all called themselves `siftlode.plex`, so neither a level pinned per logger nor E22.6's console
|
||||
filter could tell a sync line from a stream line. Derived from `__name__` there is exactly one
|
||||
name per module, it cannot drift from the file it sits in, and `test_logger_names.py` fails if a
|
||||
module hand-rolls one again.
|
||||
|
||||
`__main__` is resolved to the module it really is. The worker runs as `python -m app.worker`
|
||||
and spawns `python -m app.downloads.runner`, so in the two long-running processes `__name__`
|
||||
is `__main__` — taken at its word, both would log as the bare root. `-m` records the real name
|
||||
on the module's spec; a plain script has none and stays the root."""
|
||||
if name == "__main__":
|
||||
spec = getattr(sys.modules.get("__main__"), "__spec__", None)
|
||||
name = getattr(spec, "name", None) or name
|
||||
stem = name.removeprefix("app.") if name.startswith("app.") else ""
|
||||
# logger-name-ok: this call IS the derivation
|
||||
return logging.getLogger(f"siftlode.{stem}" if stem else "siftlode")
|
||||
|
||||
|
||||
def resolve_level(name: str) -> int:
|
||||
"""Level name → number. Both ways in (the `LOG_LEVEL` env and the `log_level` admin knob) are
|
||||
already held to the `LogLevel` choices, so an unknown name here is a programming error."""
|
||||
value = logging.getLevelName(name)
|
||||
if not isinstance(value, int):
|
||||
raise ValueError(f"unknown log level {name!r}")
|
||||
return value
|
||||
|
||||
|
||||
def configure_logging(source: str) -> None:
|
||||
"""Configure logging for THIS process. Call once, first thing, in every entry point:
|
||||
`main.py` (api), `worker.main`, `downloads/runner.main`, `alembic/env.py`.
|
||||
|
||||
`source` names the process in every line. The level starts at `LOG_LEVEL` (the bootstrap default);
|
||||
the runtime admin knob goes through `set_level()` instead, so changing it needs no restart.
|
||||
|
||||
Our loggers deliberately keep `propagate=True` and own no handler: everything reaches the single
|
||||
root StreamHandler. That is what keeps `caplog` working in tests, and it means a new logger name
|
||||
is configured the moment it is created."""
|
||||
global _source
|
||||
_source = source
|
||||
formatter = settings.log_format
|
||||
logging.config.dictConfig(
|
||||
{
|
||||
"version": 1,
|
||||
# Never disable loggers created at import time (module-level `getLogger` calls run
|
||||
# before this function does).
|
||||
"disable_existing_loggers": False,
|
||||
"formatters": {
|
||||
"json": {"()": f"{__name__}.JsonFormatter"},
|
||||
"text": {"()": f"{__name__}.TextFormatter"},
|
||||
},
|
||||
"filters": {
|
||||
"context": {"()": f"{__name__}.ContextFilter"},
|
||||
"gate": {"()": f"{__name__}.LevelGate"},
|
||||
},
|
||||
"handlers": {
|
||||
"stdout": {
|
||||
"class": "logging.StreamHandler",
|
||||
"stream": sys.stdout,
|
||||
"formatter": formatter,
|
||||
"filters": ["context", "gate"],
|
||||
}
|
||||
},
|
||||
"root": {"level": resolve_level(settings.log_level), "handlers": ["stdout"]},
|
||||
"loggers": {name: {"level": lvl} for name, lvl in _THIRD_PARTY_LEVELS.items()},
|
||||
}
|
||||
)
|
||||
set_level(settings.log_level)
|
||||
|
||||
|
||||
def set_level(name: str) -> None:
|
||||
"""Change the running process's level (the `log_level` admin knob). Third-party loggers keep
|
||||
their pinned levels, so turning on DEBUG shows OUR detail, not SQLAlchemy's."""
|
||||
global _base_level
|
||||
_base_level = resolve_level(name)
|
||||
_apply_root_level(force=True)
|
||||
|
||||
|
||||
def current_level() -> str:
|
||||
"""The configured level — not the root logger's, which an active elevation may have lowered."""
|
||||
return logging.getLevelName(_base_level or logging.getLogger().getEffectiveLevel())
|
||||
|
||||
|
||||
# --- per-user elevation (E22 S5) -----------------------------------------------------------
|
||||
# The base level (`LOG_LEVEL`, then the admin knob). NOTSET until configure_logging runs — a unit
|
||||
# test, a script — so the gate lets everything through there, as it did before the gate existed.
|
||||
_base_level = logging.NOTSET
|
||||
# → (level number, expiry) for every elevation this process has seen and not yet expired: a user's
|
||||
# own, by user id, and a debug session's (E16), by (user id, session id) — its requests alone carry
|
||||
# its level, so it is tracked apart from the user's own requests.
|
||||
_ElevationKey = int | tuple[int, str]
|
||||
_elevations: dict[_ElevationKey, tuple[int, datetime]] = {}
|
||||
_elevations_lock = threading.Lock()
|
||||
_root_applied: int | None = None
|
||||
|
||||
|
||||
def _track_elevation(key: _ElevationKey, level: int | None, until: datetime | None) -> None:
|
||||
with _elevations_lock:
|
||||
if level is None or until is None:
|
||||
_elevations.pop(key, None)
|
||||
else:
|
||||
_elevations[key] = (level, until)
|
||||
active = bool(_elevations)
|
||||
if active or _root_applied != _base_level:
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def _apply_root_level(force: bool = False) -> None:
|
||||
"""Root level = the lowest of the base and every unexpired elevation (expired ones are pruned
|
||||
here). `setLevel` clears every logger's level cache, so it runs only when the number changes."""
|
||||
global _root_applied
|
||||
if _base_level == logging.NOTSET:
|
||||
return # not configured: the root keeps whatever the process set up
|
||||
now = datetime.now(timezone.utc)
|
||||
# The level is SET under the lock too, not just computed: two threads that each computed a
|
||||
# different answer would otherwise apply them in either order, and the older one could win —
|
||||
# the root back at the base while an elevation is still active.
|
||||
with _elevations_lock:
|
||||
for key in [k for k, (_, until) in _elevations.items() if until <= now]:
|
||||
del _elevations[key]
|
||||
lowest = min([_base_level, *(lvl for lvl, _ in _elevations.values())])
|
||||
if force or lowest != _root_applied:
|
||||
logging.getLogger().setLevel(lowest)
|
||||
_root_applied = lowest
|
||||
|
||||
|
||||
# --- erased accounts ------------------------------------------------------------------------
|
||||
# `auth.purge_user` deletes an account's `log_events` rows, but a line about that account can still
|
||||
# be on its way: the erasing request's own access line is logged AFTER the handler committed, a queued
|
||||
# row waits for the sink's writer, a background task logs later. Those rows are stored without the
|
||||
# account (`logsink` asks here). Remembered for an hour in this process — a line later than that has
|
||||
# no request left to come from, and the retention job is the backstop for any other process.
|
||||
ERASED_MEMORY_SECONDS = 3600.0
|
||||
_erased: dict[int, float] = {}
|
||||
_erased_lock = threading.Lock()
|
||||
# Held by whoever checks rows against `is_erased` AND writes them (`erasure_guard`). `forget_user`
|
||||
# takes it too, so an erasure waits for a write that already passed the check to commit — and the
|
||||
# purge's DELETE, which runs after `forget_user`, then removes those rows as well. Without it a row
|
||||
# checked a moment before the erasure could be inserted a moment after the DELETE.
|
||||
_erasure_guard = threading.Lock()
|
||||
|
||||
|
||||
def erasure_guard() -> threading.Lock:
|
||||
"""Hold this around "drop erased accounts from these rows, insert them, commit"."""
|
||||
return _erasure_guard
|
||||
|
||||
|
||||
def forget_user(user_id: int) -> None:
|
||||
"""Mark `user_id` erased: no line logged about it from now on is stored under its id. Call BEFORE
|
||||
deleting its rows (see `_erasure_guard`)."""
|
||||
now = time.monotonic()
|
||||
with _erasure_guard, _erased_lock:
|
||||
for stale in [u for u, at in _erased.items() if now - at > ERASED_MEMORY_SECONDS]:
|
||||
del _erased[stale]
|
||||
_erased[user_id] = now
|
||||
with _elevations_lock:
|
||||
for key in [
|
||||
k for k in _elevations if k == user_id or (isinstance(k, tuple) and k[0] == user_id)
|
||||
]:
|
||||
del _elevations[key]
|
||||
# That pop can have removed the lowest elevation, and the root level is the lowest of the base
|
||||
# and every unexpired one. Recomputed here rather than left to whichever later request happens
|
||||
# to call `_track_elevation` — the invariant `_apply_root_level` documents holds at every pop.
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def end_session_elevation(session_id: str) -> None:
|
||||
"""A debug session was stopped: its elevation ends now, not at its cap."""
|
||||
with _elevations_lock:
|
||||
for key in [k for k in _elevations if isinstance(k, tuple) and k[1] == session_id]:
|
||||
del _elevations[key]
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def is_erased(user_id: int | None) -> bool:
|
||||
if user_id is None:
|
||||
return False
|
||||
with _erased_lock:
|
||||
at = _erased.get(user_id)
|
||||
return at is not None and time.monotonic() - at <= ERASED_MEMORY_SECONDS
|
||||
|
||||
|
||||
def current_source() -> str:
|
||||
"""The process name `configure_logging` was given (api, worker, runner, alembic)."""
|
||||
return _source
|
||||
|
||||
|
||||
# --- request logging -----------------------------------------------------------------------
|
||||
# A client may supply its own request id so a browser log line and a server line join up (E22 S4).
|
||||
# Accepted only in this shape: it ends up in a log field, and an unbounded header would be a way to
|
||||
# write arbitrary text into the log.
|
||||
_REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,64}$")
|
||||
|
||||
# Paths whose success is logged at DEBUG rather than INFO: the container healthcheck runs every 15s
|
||||
# (5.7k lines/day saying nothing), hashed static assets are served by the hundred per page load, and
|
||||
# the admin log console reads the stored log — its live tail every few seconds, so at INFO the log
|
||||
# would fill with the console watching itself. A FAILURE on any of them still logs at WARNING/ERROR
|
||||
# — this lowers the floor, it does not hide.
|
||||
_QUIET_PREFIXES = ("/healthz", "/assets/", "/static/", "/api/admin/logs")
|
||||
|
||||
# The reads the frontend POLLS (a `useLiveQuery`, the `me` heartbeat, the sync status, the subtitle
|
||||
# progress), by route template, GET only — logged at DEBUG like the prefixes above. One open tab asked
|
||||
# for the two unread counts, `/api/me` and the download index every 30 s, and at INFO those were ~95%
|
||||
# of the stored log (measured on prod 2026-09-19): 11k lines a day per tab burying every line someone
|
||||
# caused. The same route answered with a failure still logs at WARNING/ERROR.
|
||||
_POLLED_ROUTES = frozenset(
|
||||
{
|
||||
"/api/me",
|
||||
"/api/me/notifications/unread_count",
|
||||
"/api/messages/unread_count",
|
||||
"/api/downloads",
|
||||
"/api/downloads/index",
|
||||
"/api/downloads/usage",
|
||||
"/api/sync/my-status",
|
||||
"/api/admin/scheduler",
|
||||
"/api/admin/downloads",
|
||||
"/api/admin/downloads/storage",
|
||||
"/api/plex/subtitle/{rating_key}/{sub_ord}/status",
|
||||
}
|
||||
)
|
||||
|
||||
# Not `siftlode.logs`: this is the replacement for uvicorn's ACCESS log, and `siftlode.http` is
|
||||
# what an operator pins a level on when they want request lines and nothing else. The module it
|
||||
# happens to live in is not what the name is about.
|
||||
_request_log = logging.getLogger("siftlode.http") # logger-name-ok: named for the traffic
|
||||
|
||||
|
||||
async def log_requests(request, call_next):
|
||||
"""The single access-log line, replacing uvicorn's.
|
||||
|
||||
What changes versus `uvicorn.access`: the ROUTE TEMPLATE instead of the concrete path (so
|
||||
`/api/videos/{video_id}` groups), never the query string (it carried `?account=`, and at TRACE
|
||||
the redacted query belongs in attrs, not in the URL), a request id echoed back in
|
||||
`X-Request-Id` (with the template in `X-Route`), the duration, and the account — read from `request.state`, which the auth
|
||||
dependency fills, because this middleware sits outside both the session middleware and the
|
||||
downstream task's context."""
|
||||
incoming = request.headers.get("x-request-id", "")
|
||||
rid = incoming if _REQUEST_ID_RE.match(incoming) else uuid.uuid4().hex[:16]
|
||||
token = _request_id.set(rid)
|
||||
scope = _RequestScope()
|
||||
scope_token = _scope.set(scope)
|
||||
request.state.request_id = rid
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
# The unhandled-exception path: FastAPI's ServerErrorMiddleware re-raises after its own
|
||||
# 500, so without this the ONE request that most needs a log line is the one with no
|
||||
# access-log entry at all.
|
||||
_request_log.exception(
|
||||
"request failed",
|
||||
extra={
|
||||
**event(
|
||||
"http.request",
|
||||
method=request.method,
|
||||
path=_route_template(request),
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 1),
|
||||
),
|
||||
**_request_owner(request, logging.ERROR),
|
||||
},
|
||||
)
|
||||
_scope.reset(scope_token)
|
||||
_request_id.reset(token)
|
||||
raise
|
||||
duration_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
status = response.status_code
|
||||
quiet = request.url.path.startswith(_QUIET_PREFIXES) or (
|
||||
request.method == "GET" and _route_template(request) in _POLLED_ROUTES
|
||||
)
|
||||
if status >= 500:
|
||||
level = logging.ERROR
|
||||
elif status >= 400:
|
||||
level = logging.WARNING
|
||||
else:
|
||||
level = logging.DEBUG if quiet else logging.INFO
|
||||
attrs: dict[str, Any] = {
|
||||
"method": request.method,
|
||||
"path": _route_template(request),
|
||||
"status": status,
|
||||
"duration_ms": duration_ms,
|
||||
}
|
||||
# Teljes (TRACE) the user consented to: the query string too — redacted, and in attrs only, never
|
||||
# in the message.
|
||||
if request.url.query and _traced(scope):
|
||||
attrs["query"] = redact_query(request.url.query)
|
||||
_request_log.log(
|
||||
level,
|
||||
"%s %s %s",
|
||||
request.method,
|
||||
_route_template(request),
|
||||
status,
|
||||
extra={**event("http.request", **attrs), **_request_owner(request, level)},
|
||||
)
|
||||
response.headers["X-Request-Id"] = rid
|
||||
# The browser logs its own copy of this line (frontend lib/http.ts) and must pick the same level:
|
||||
# it learns "this one is quiet" here instead of keeping a second copy of the polled-route list.
|
||||
if level == logging.DEBUG:
|
||||
response.headers["X-Log-Quiet"] = "1"
|
||||
# The template, so the browser's copy of this line groups the same way (frontend lib/http.ts).
|
||||
# Only a MATCHED route's: a template never carries a path parameter's value (a share token stays
|
||||
# `{token}`), while the raw-path fallback of an unmatched request could — the browser then
|
||||
# templates the path itself.
|
||||
matched = _matched_route(request)
|
||||
if matched:
|
||||
response.headers["X-Route"] = matched
|
||||
_scope.reset(scope_token)
|
||||
_request_id.reset(token)
|
||||
return response
|
||||
|
||||
|
||||
def consented_trace() -> bool:
|
||||
"""Whether the current request's OWN user turned Teljes (TRACE) on — the gate for a line that
|
||||
carries what they typed or what was quoted back to them. The level alone is not enough: an
|
||||
admin running the whole process at TRACE would pass it for every user, and a scheduler thread,
|
||||
which has no request, for none of them in particular. Guard such a line with this, then log it
|
||||
at TRACE."""
|
||||
scope = _scope.get()
|
||||
return scope is not None and _traced(scope)
|
||||
|
||||
|
||||
def _traced(scope: _RequestScope) -> bool:
|
||||
"""Whether this request's query string is logged: only under its OWN user's Teljes consent. An
|
||||
admin setting the whole process to TRACE gets more detail, not every user's search terms — the
|
||||
privacy policy promises those are kept only for a user who agreed to it."""
|
||||
return scope.level is not None and scope.level <= TRACE
|
||||
|
||||
|
||||
# Anonymous request lines kept per client per minute. Signed-in traffic is not capped: it is a known
|
||||
# account and exactly what an admin needs to see. Anonymous traffic is where the volume can come from
|
||||
# nobody in particular — a scanner walking `/wp-login.php`, a bot on a share link — and each of its
|
||||
# 404s is a WARNING that would otherwise land in `log_events` without limit. The client key is an
|
||||
# in-memory counter key only; no address is ever written to a log line or a row.
|
||||
ANONYMOUS_LINES_PER_MINUTE = 60
|
||||
_anonymous_lines = RateLimiter(ANONYMOUS_LINES_PER_MINUTE, 60.0)
|
||||
|
||||
|
||||
def _request_owner(request, level: int) -> dict[str, Any]:
|
||||
"""`extra=` fields naming who the request belongs to, and whether its line may be stored. The
|
||||
account comes from `request.state` (filled by the auth dependency), so it lands in the line's
|
||||
top-level `user_id` — the correlation field — rather than in `attrs`."""
|
||||
user_id = getattr(request.state, "user_id", None)
|
||||
if user_id is not None:
|
||||
return {"user_id": user_id}
|
||||
if level < logging.INFO:
|
||||
return {} # a quiet line is never stored anyway — don't spend the client's allowance on it
|
||||
return {} if _anonymous_lines.allow(client_key(request)) else {PERSIST: False}
|
||||
|
||||
|
||||
def _matched_route(request) -> str | None:
|
||||
"""The matched route's template (`/api/videos/{video_id}`), or None when nothing matched."""
|
||||
return getattr(request.scope.get("route"), "path", None)
|
||||
|
||||
|
||||
def _route_template(request) -> str:
|
||||
"""The matched route's template, else the raw path for a 404 or a static file. Never the query
|
||||
string."""
|
||||
return _matched_route(request) or request.url.path
|
||||
@@ -0,0 +1,367 @@
|
||||
"""The `log_events` sink (E22 S3): the stored copy of the log the diagnostics console reads.
|
||||
|
||||
stdout stays the complete log (docker keeps 3×50 MB of it); this table keeps the part worth querying
|
||||
for `log_retention_days`:
|
||||
|
||||
* every WARNING and above,
|
||||
* the `http.request` line of every request at INFO or above (not the DEBUG-level health checks and
|
||||
static files),
|
||||
* every line of a request whose user raised their diagnostics level (E22 S5 — `logs.LevelGate`
|
||||
marks them), which is the detail they consented to have kept,
|
||||
* nothing a caller marked `extra={logs.PERSIST: False}` — how the request log caps anonymous
|
||||
traffic per client.
|
||||
|
||||
What the table must never do is slow down or break the code that logs. So a line is redacted and
|
||||
turned into a row on the CALLER's thread (the same `logs.record_fields` the stdout line is built
|
||||
from), put on a bounded queue without blocking, and written in batches by one background thread
|
||||
on its own connection — never inside a request's transaction. A full queue drops the row and
|
||||
counts it; a failed write drops the batch and says so on stderr. Both are reported rather than
|
||||
retried: a log that retries into a sick database is how a small outage becomes a big one.
|
||||
|
||||
Flood control is the second half. On 2026-09-16 a YouTube RSS outage made prod log 11k near-identical
|
||||
warnings in four hours. stdout keeps them all; the table keeps `REPEATS_PER_MINUTE` of each message
|
||||
template per minute and one `log.repeats_suppressed` row saying how many more there were.
|
||||
|
||||
Core layer: imports `logs`, `db`, `models`, `config` — nothing above them.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, insert
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.db import engine
|
||||
from app.models import LogEvent
|
||||
from app.utils import iso
|
||||
|
||||
QUEUE_MAX = 10_000
|
||||
BATCH_MAX = 500
|
||||
FLUSH_SECONDS = 1.0
|
||||
REPEATS_PER_MINUTE = 30
|
||||
REPEAT_WINDOW_SECONDS = 60.0
|
||||
# Distinct templates tracked per window. Past this many, a new template is stored without counting
|
||||
# — the limiter must not become the unbounded thing it exists to prevent.
|
||||
_REPEAT_KEYS_MAX = 2_000
|
||||
|
||||
# Column widths from models.LogEvent. A value past its column would fail the WHOLE batch it rides in.
|
||||
_LIMITS = {
|
||||
"level": 10,
|
||||
"source": 16,
|
||||
"logger": 128,
|
||||
"event": 64,
|
||||
"request_id": 64,
|
||||
"debug_session_id": 64,
|
||||
"version": 64,
|
||||
"msg": 8_000,
|
||||
"exc": 32_000,
|
||||
}
|
||||
|
||||
_log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def should_store(record: logging.LogRecord) -> bool:
|
||||
"""Whether this record belongs in `log_events` (see the module docstring)."""
|
||||
if getattr(record, logs.PERSIST, True) is False:
|
||||
return False
|
||||
if record.levelno >= logging.WARNING or getattr(record, "elevated", False):
|
||||
return True
|
||||
return getattr(record, "event", None) == "http.request" and record.levelno >= logging.INFO
|
||||
|
||||
|
||||
def _without_nul(value: Any) -> Any:
|
||||
"""`value` with every NUL character replaced, in keys and nested values alike. Postgres refuses
|
||||
NUL in text and `\\u0000` in JSONB, and a refused row fails the WHOLE batch it rides in — every
|
||||
user's rows with it. Anything logged can carry one (`GET /assets/%00` logs the decoded path)."""
|
||||
if isinstance(value, str):
|
||||
return value.replace("\x00", "�")
|
||||
if isinstance(value, dict):
|
||||
return {_without_nul(k): _without_nul(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_without_nul(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def to_row(fields: dict[str, Any]) -> dict[str, Any]:
|
||||
"""`logs.record_fields` output → a `log_events` row: known columns only, text clipped to its
|
||||
column and free of NUL, attrs made JSON-safe (anything exotic becomes its `str`, as on stdout)."""
|
||||
row = {
|
||||
column: fields.get(column) for column in LogEvent.__table__.columns.keys() if column != "id"
|
||||
}
|
||||
for column, limit in _LIMITS.items():
|
||||
value = row.get(column)
|
||||
if isinstance(value, str):
|
||||
value = _without_nul(value)
|
||||
row[column] = value[: limit - 1] + "…" if len(value) > limit else value
|
||||
if row["attrs"] is not None:
|
||||
row["attrs"] = _without_nul(
|
||||
json.loads(json.dumps(row["attrs"], ensure_ascii=False, default=str))
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def without_erased(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""The rows, with the account taken off any whose account was erased meanwhile
|
||||
(`logs.forget_user`). Checked right before the INSERT, under `logs.erasure_guard()` held until
|
||||
the commit: a row built before the erasure can still be waiting for its batch."""
|
||||
for row in rows:
|
||||
if logs.is_erased(row.get("user_id")):
|
||||
row["user_id"] = None
|
||||
return rows
|
||||
|
||||
|
||||
def _notice(code: str, msg: str, **attrs: Any) -> dict[str, Any]:
|
||||
"""A row the sink writes about ITSELF. Built directly, not logged: logging it would route it back
|
||||
through this sink (and the writer thread's own records are deliberately ignored)."""
|
||||
return to_row(
|
||||
{
|
||||
"ts": datetime.now(timezone.utc),
|
||||
"level": "WARNING",
|
||||
"sev": logs.SEVERITY["WARNING"],
|
||||
"source": logs.current_source(),
|
||||
"logger": _log.name,
|
||||
"event": code,
|
||||
"msg": msg,
|
||||
"attrs": attrs,
|
||||
"version": settings.app_version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_DIGITS = re.compile(r"\d+")
|
||||
|
||||
|
||||
def _template(record: logging.LogRecord) -> str:
|
||||
"""The flood limiter's grouping key: the message template. Except when the template is a bare
|
||||
`%s` — a pass-through for someone else's text (yt-dlp's lines, `formats._YdlLog`), where the
|
||||
template says nothing and every distinct line would share one bucket. Then the text itself
|
||||
groups, with its numbers folded, so `Retrying fragment 5 (1/10)` and `(2/10)` are still one
|
||||
repeat and a different warning is not crowded out by them."""
|
||||
msg = str(record.msg)
|
||||
if msg != "%s" or not record.args:
|
||||
return msg
|
||||
return _DIGITS.sub("#", record.getMessage())[:200]
|
||||
|
||||
|
||||
class _Repeats:
|
||||
"""Per-template admission for one minute at a time. Used by the writer thread only (unlocked)."""
|
||||
|
||||
def __init__(self, per_window: int, window_seconds: float):
|
||||
self.per_window = per_window
|
||||
self.window = window_seconds
|
||||
self.started = time.monotonic()
|
||||
self.seen: dict[tuple[str, str, int], int] = {}
|
||||
|
||||
def admit(self, key: tuple[str, str, int] | None) -> bool:
|
||||
if key is None:
|
||||
return True
|
||||
count = self.seen.get(key)
|
||||
if count is None and len(self.seen) >= _REPEAT_KEYS_MAX:
|
||||
return True
|
||||
self.seen[key] = (count or 0) + 1
|
||||
return self.seen[key] <= self.per_window
|
||||
|
||||
def roll(self, now: float, *, final: bool = False) -> list[dict[str, Any]]:
|
||||
"""Close the window if it is over — or, `final`, because the process is: one notice per
|
||||
template that went past the limit. Without `final` a process that lives under a minute
|
||||
(every download child) would drop its repeats and never say so."""
|
||||
if not final and now - self.started < self.window:
|
||||
return []
|
||||
notices = [
|
||||
_notice(
|
||||
"log.repeats_suppressed",
|
||||
f"{count - self.per_window} more like this were not stored",
|
||||
logger=logger,
|
||||
template=logs.redact_text(template),
|
||||
suppressed=count - self.per_window,
|
||||
window_seconds=int(self.window),
|
||||
)
|
||||
for (logger, template, _level), count in self.seen.items()
|
||||
if count > self.per_window
|
||||
]
|
||||
self.seen.clear()
|
||||
self.started = now
|
||||
return notices
|
||||
|
||||
|
||||
class LogSink(logging.Handler):
|
||||
"""The handler plus its writer. `start()` runs the writer thread; a test drives `write_pending()`
|
||||
directly instead, on its own thread, with no timing involved."""
|
||||
|
||||
def __init__(self, bind: Engine = engine):
|
||||
super().__init__(level=logging.NOTSET)
|
||||
# Correlation fields are contextvars: capture them now, on the thread that logged.
|
||||
self.addFilter(logs.ContextFilter())
|
||||
self._bind = bind
|
||||
self._queue: queue.Queue[tuple[tuple[str, str, int] | None, dict[str, Any]]] = queue.Queue(
|
||||
QUEUE_MAX
|
||||
)
|
||||
self._dropped = 0
|
||||
self._dropped_lock = threading.Lock()
|
||||
self._repeats = _Repeats(REPEATS_PER_MINUTE, REPEAT_WINDOW_SECONDS)
|
||||
self._stopping = threading.Event()
|
||||
self._writer: threading.Thread | None = None
|
||||
|
||||
# --- caller side ---------------------------------------------------------------------------
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
if threading.current_thread() is self._writer:
|
||||
return # the writer's own trouble (a driver warning mid-insert) must not feed itself
|
||||
if not should_store(record):
|
||||
return
|
||||
try:
|
||||
row = to_row(logs.record_fields(record))
|
||||
# silent-ok: `handleError` is the report — logging's own out-of-band channel to stderr.
|
||||
# A log call here would be handed to this handler, which is the one that just failed.
|
||||
except Exception: # noqa: BLE001 — a record we cannot turn into a row is reported, not raised
|
||||
self.handleError(record)
|
||||
return
|
||||
# Request lines are one per request by nature; only the repeated SAME message is a flood.
|
||||
key = (
|
||||
None
|
||||
if row["event"] == "http.request"
|
||||
else (record.name, _template(record), record.levelno)
|
||||
)
|
||||
try:
|
||||
self._queue.put_nowait((key, row))
|
||||
# silent-ok: counted, not logged — the count goes out as the `log.sink.dropped` notice row
|
||||
# the writer emits, which is the one report that does not feed this queue again.
|
||||
except queue.Full:
|
||||
with self._dropped_lock:
|
||||
self._dropped += 1
|
||||
|
||||
# --- writer side ---------------------------------------------------------------------------
|
||||
def write_pending(self, wait: float = 0.0, *, final: bool = False) -> int:
|
||||
"""Write one batch: whatever is queued (up to BATCH_MAX), waiting up to `wait` seconds for the
|
||||
first item. Returns the number of rows written, notices included. `final` closes the
|
||||
repeat window early — the last batch before the process exits."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
item = self._queue.get(timeout=wait) if wait > 0 else self._queue.get_nowait()
|
||||
# silent-ok: an idle sink. Nothing failed; there was simply nothing waiting.
|
||||
except queue.Empty:
|
||||
item = None
|
||||
while item is not None:
|
||||
key, row = item
|
||||
if self._repeats.admit(key):
|
||||
rows.append(row)
|
||||
if len(rows) >= BATCH_MAX:
|
||||
break
|
||||
try:
|
||||
item = self._queue.get_nowait()
|
||||
# silent-ok: the batch ends here because the queue ran dry, which is the normal end.
|
||||
except queue.Empty:
|
||||
item = None
|
||||
rows.extend(self._repeats.roll(time.monotonic(), final=final))
|
||||
with self._dropped_lock:
|
||||
dropped, self._dropped = self._dropped, 0
|
||||
if dropped:
|
||||
rows.append(
|
||||
_notice("log.sink.dropped", f"{dropped} log events were dropped", dropped=dropped)
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
try:
|
||||
# The connection is checked out BEFORE the guard, and every request path takes them in
|
||||
# that order too (its session already holds a connection when it reaches the guard). The
|
||||
# other way round, this thread would hold a process-global lock while waiting up to
|
||||
# `pool_timeout` for a connection, and the requests that could return one are the ones
|
||||
# blocked on the lock. The guard still spans the check, the insert and the commit.
|
||||
with self._bind.connect() as conn, logs.erasure_guard(), conn.begin():
|
||||
conn.execute(insert(LogEvent), without_erased(rows))
|
||||
except Exception as exc: # noqa: BLE001 — the database is down or refused the batch
|
||||
# silent-ok / noqa T201: the sink's OWN failure. A log call here would be handed to
|
||||
# this sink, which is the thing that just failed — straight into a loop. stderr is
|
||||
# the one way out that does not re-enter.
|
||||
print( # noqa: T201
|
||||
f"log sink: could not store {len(rows)} log events: "
|
||||
f"{logs.redact_text(str(exc))[:300]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
return len(rows)
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stopping.is_set():
|
||||
self.write_pending(wait=FLUSH_SECONDS)
|
||||
while self.write_pending(): # drain what arrived before the stop
|
||||
pass
|
||||
self.write_pending(final=True) # and the repeat notices of a window cut short
|
||||
|
||||
def start(self) -> None:
|
||||
self._writer = threading.Thread(target=self._run, name="log-sink", daemon=True)
|
||||
self._writer.start()
|
||||
|
||||
def stop(self, timeout: float = 5.0) -> None:
|
||||
self._stopping.set()
|
||||
if self._writer is not None:
|
||||
self._writer.join(timeout)
|
||||
|
||||
|
||||
_sink: LogSink | None = None
|
||||
|
||||
|
||||
def start() -> None:
|
||||
"""Attach the sink to the root logger and start its writer. Call after `logs.configure_logging`
|
||||
(which replaces the root handlers) and once the schema exists. A no-op when `LOG_DB_SINK` is off
|
||||
or it is already running."""
|
||||
global _sink
|
||||
if _sink is not None or not settings.log_db_sink:
|
||||
return
|
||||
_sink = LogSink()
|
||||
# The same level gate as stdout: the root may be lowered for one user's elevated requests, and
|
||||
# everyone else's detail must not reach the table through that gap either.
|
||||
_sink.addFilter(logs.LevelGate())
|
||||
logging.getLogger().addHandler(_sink)
|
||||
_sink.start()
|
||||
# A short-lived process (the download child) exits without a lifespan: flush on the way out.
|
||||
atexit.register(stop)
|
||||
|
||||
|
||||
def stop() -> None:
|
||||
"""Detach the sink and write what is still queued."""
|
||||
global _sink
|
||||
if _sink is None:
|
||||
return
|
||||
logging.getLogger().removeHandler(_sink)
|
||||
_sink.stop()
|
||||
_sink = None
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Stage the deletion of rows older than `retention_days` (at least one day — stored logs always
|
||||
expire). The caller commits. Returns the number of rows removed."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, retention_days))
|
||||
result = db.execute(delete(LogEvent).where(LogEvent.ts < cutoff))
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def serialize(row: LogEvent) -> dict:
|
||||
"""A stored row as the API sends it — one shape for every reader: the admin console (E22.6) and
|
||||
a debug session's evidence list (E16)."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"ts": iso(row.ts),
|
||||
"level": row.level,
|
||||
"source": row.source,
|
||||
"logger": row.logger,
|
||||
"event": row.event,
|
||||
"msg": row.msg,
|
||||
"attrs": row.attrs,
|
||||
"exc": row.exc,
|
||||
"request_id": row.request_id,
|
||||
"user_id": row.user_id,
|
||||
"job_id": row.job_id,
|
||||
"debug_session_id": row.debug_session_id,
|
||||
"version": row.version,
|
||||
}
|
||||
@@ -175,7 +175,7 @@ MAIL_TEXT: dict[str, dict[str, str]] = {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Someone has opened up Siftlode for you — a quiet, self-hosted reader for your YouTube "
|
||||
"subscriptions. Every upload from the channels you follow lands in one feed you can "
|
||||
"subscriptions. Every upload from the channels you follow lands in one Feed you can "
|
||||
"actually filter and sort, with no algorithm deciding what you see, and no Shorts or "
|
||||
"livestreams unless you want them.\n\n"
|
||||
"{next_step}\n\n"
|
||||
@@ -186,7 +186,7 @@ MAIL_TEXT: dict[str, dict[str, str]] = {
|
||||
"Szia,\n\n"
|
||||
"Valaki megnyitotta előtted a Siftlode-ot — ez egy csendes, saját üzemeltetésű olvasó "
|
||||
"a YouTube-feliratkozásaidhoz. A követett csatornák minden feltöltése egyetlen "
|
||||
"hírfolyamba érkezik, amit tényleg tudsz szűrni és rendezni; nincs algoritmus, ami "
|
||||
"Feedbe érkezik, amit tényleg tudsz szűrni és rendezni; nincs algoritmus, ami "
|
||||
"eldönti, mit látsz, és nincs Shorts vagy élő adás, hacsak nem kéred.\n\n"
|
||||
"{next_step}\n\n"
|
||||
"Nem sürget semmi — az ajtó nyitva marad.\n\n"
|
||||
|
||||
+133
-27
@@ -1,38 +1,45 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import math
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
|
||||
# When started via uvicorn --log-config the "siftlode" logger is already configured
|
||||
# (see log_config.json). This block is a timestamped fallback for other entrypoints
|
||||
# (tests, scripts) so our logs are never silently dropped.
|
||||
_siftlode_logger = logging.getLogger("siftlode")
|
||||
if not _siftlode_logger.handlers:
|
||||
_handler = logging.StreamHandler(sys.stdout)
|
||||
_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s"))
|
||||
_siftlode_logger.addHandler(_handler)
|
||||
_siftlode_logger.setLevel(logging.INFO)
|
||||
_siftlode_logger.propagate = False
|
||||
# Logging is configured HERE, before the route modules are imported, so every logger they create at
|
||||
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
||||
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
||||
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
||||
from app import logs
|
||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||
|
||||
log = logging.getLogger("siftlode.app")
|
||||
configure_logging("api")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.exc import DataError
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from app import bodylimit
|
||||
from app import logsink
|
||||
from app import state
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.plex import stream as plex_stream
|
||||
from app.plex import subtitles as plex_subtitles
|
||||
from app.routes import (
|
||||
admin,
|
||||
admin_logs as admin_logs_routes,
|
||||
audit as audit_routes,
|
||||
auth as auth_routes,
|
||||
channels,
|
||||
config as config_routes,
|
||||
debug_sessions as debug_sessions_routes,
|
||||
diagnostics as diagnostics_routes,
|
||||
discover as discover_routes,
|
||||
downloads,
|
||||
feed,
|
||||
@@ -58,16 +65,28 @@ from app.scheduler import shutdown_scheduler, start_scheduler
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
log.info("Siftlode starting up")
|
||||
# The stored copy of the log (log_events). The entrypoint has already migrated the schema.
|
||||
logsink.start()
|
||||
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
||||
# the wizard URL so the operator can open it (the app runs in setup mode until they finish).
|
||||
with SessionLocal() as db:
|
||||
# The admin's runtime level (sysconfig `log_level`) wins over the LOG_LEVEL bootstrap
|
||||
# default, so a level chosen in the UI survives a restart without touching the environment.
|
||||
# Best-effort on purpose: a LOGGING knob must never be the thing that stops the app from
|
||||
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
||||
try:
|
||||
set_level(sysconfig.get_str(db, "log_level"))
|
||||
except Exception: # noqa: BLE001 — any DB/config failure keeps the bootstrap level
|
||||
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
||||
# and `%s` of the exception does not say which of them raised.
|
||||
log.exception("could not read the stored log level; staying at %s", current_level())
|
||||
if not state.is_configured(db):
|
||||
token = state.rotate_setup_token(db)
|
||||
url = f"{settings.app_base}/setup?token={token}"
|
||||
log.warning(
|
||||
"FIRST-RUN SETUP REQUIRED — this instance isn't configured yet.\n"
|
||||
" Open the install wizard (internal access only):\n %s",
|
||||
url,
|
||||
url, # log-pii-ok: this instance's OWN setup URL, and `?token=` is redacted
|
||||
)
|
||||
# No HLS session survives a restart (they live in this process), so anything still on disk is
|
||||
# a crashed run's segments — sweep them or they accumulate forever. FIRE AND FORGET, not
|
||||
@@ -77,6 +96,7 @@ async def lifespan(app: FastAPI):
|
||||
# alone doesn't buy that; not waiting does. The sweep only touches directories no live session
|
||||
# owns, so racing it against early playback is safe.
|
||||
sweep = asyncio.create_task(_sweep_hls_orphans())
|
||||
reaper = asyncio.create_task(_reap_hls_sessions())
|
||||
start_scheduler()
|
||||
try:
|
||||
yield
|
||||
@@ -86,10 +106,37 @@ async def lifespan(app: FastAPI):
|
||||
# pending ("Task was destroyed but it is pending!" — noise that reads like a bug during
|
||||
# triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own
|
||||
# bookkeeping honest.
|
||||
sweep.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await sweep
|
||||
for task in (sweep, reaper):
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
shutdown_scheduler()
|
||||
# Kills a running subtitle extraction: its worker would otherwise hold interpreter exit for
|
||||
# the rest of the read (see plex_subtitles.shutdown).
|
||||
plex_subtitles.shutdown()
|
||||
logsink.stop() # last: write what the shutdown itself just logged
|
||||
|
||||
|
||||
# How often idle HLS sessions are reaped (see plex/stream.py `reap_idle`).
|
||||
_HLS_REAP_EVERY_S = 60.0
|
||||
|
||||
|
||||
async def _reap_hls_sessions() -> None:
|
||||
"""The periodic reaper plex/stream.py's docstring promises. `reap_idle` used to run only inside
|
||||
the NEXT session request, so after the last playback of the day an abandoned session — its
|
||||
ffmpeg still working through the film, its directory a whole segmented copy of it — lived until
|
||||
somebody pressed play again or the process restarted.
|
||||
|
||||
A FINISHED session is reaped on the short rule here, not kept for the full idle window: an open
|
||||
player checkpoints every 10 s and that route touches the session, so a paused viewer is no longer
|
||||
invisible to this pass (see `reap_idle`). Keeping every finished session for ten minutes instead
|
||||
held one whole segmented copy of a film per item on the download volume (round 5)."""
|
||||
while True:
|
||||
await asyncio.sleep(_HLS_REAP_EVERY_S)
|
||||
try:
|
||||
await asyncio.to_thread(plex_stream.reap_idle)
|
||||
except Exception:
|
||||
log.exception("HLS session reap failed")
|
||||
|
||||
|
||||
async def _sweep_hls_orphans() -> None:
|
||||
@@ -108,6 +155,34 @@ async def _sweep_hls_orphans() -> None:
|
||||
|
||||
app = FastAPI(title=settings.app_name, lifespan=lifespan)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
"""FastAPI's own 422, in its own shape — except that it can be ENCODED. The JSON parser accepts
|
||||
`NaN`/`Infinity`, a float field refuses them, and the refusal echoes the value back: the default
|
||||
handler then failed to serialise its own error (JSON has no NaN) and the request became a 500
|
||||
with a traceback. A non-finite value is echoed as its name instead."""
|
||||
finite = {float: lambda x: x if math.isfinite(x) else str(x)}
|
||||
return JSONResponse(
|
||||
status_code=422, content={"detail": jsonable_encoder(exc.errors(), custom_encoder=finite)}
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(DataError)
|
||||
async def _nul_in_text(request: Request, exc: DataError) -> JSONResponse:
|
||||
"""A NUL byte in a string the request sent is a 422, on every route at once. Postgres cannot hold
|
||||
NUL in text, so the driver refuses to even send the query — and a `%00` in a query parameter or
|
||||
an escaped NUL in a JSON body (a playlist's `contains=`, a reorder's key list, a collection
|
||||
search) was a 500 with a traceback wherever it reached a comparison. Guarding each site did not
|
||||
hold: round 3 guarded the by-rating_key lookup and missed the rest. Any other DataError is left
|
||||
alone — re-raised, it stays the 500 it always was."""
|
||||
if "NUL (0x00)" not in str(exc.orig):
|
||||
raise exc
|
||||
return JSONResponse(
|
||||
status_code=422, content={"detail": {"code": "invalid_input", "reason": "nul"}}
|
||||
)
|
||||
|
||||
|
||||
app.add_middleware(
|
||||
SessionMiddleware,
|
||||
secret_key=settings.secret_key,
|
||||
@@ -115,15 +190,6 @@ app.add_middleware(
|
||||
https_only=settings.session_https_only, # Secure flag when served over HTTPS (prod)
|
||||
)
|
||||
|
||||
if settings.frontend_origin:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[settings.frontend_origin],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Paths reachable while the instance is unconfigured (setup mode). Everything else under /api or
|
||||
# /auth is locked until the wizard finishes; static/SPA always loads so the wizard page can render.
|
||||
_SETUP_OPEN_PREFIXES = ("/api/setup", "/api/version")
|
||||
@@ -143,6 +209,34 @@ async def setup_gate(request, call_next):
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# SEC-20 — the global request-body cap (app/bodylimit.py). FastAPI reads a declared body IN FULL
|
||||
# before the first dependency, so without this the anonymous login route is a free memory/time sink.
|
||||
#
|
||||
# WHERE it sits is the whole design, and it is pinned by `test_body_limit.py` rather than by this
|
||||
# comment, because `add_middleware` inverts the reading order (last registered = OUTERMOST) and a
|
||||
# stack described in prose drifts. Registered here it lands directly INSIDE CORS and directly
|
||||
# OUTSIDE the setup gate, which is the narrowest position that satisfies both constraints:
|
||||
#
|
||||
# * inside CORS and the security headers, so the 413 is decorated like every other response. A
|
||||
# refusal the browser drops for a missing Access-Control-Allow-Origin is no better than a hang,
|
||||
# and `capped_body`'s 413 one layer down IS decorated — two shapes for one fact is exactly what
|
||||
# bodylimit.py exists to avoid.
|
||||
# * outside the setup gate, the session and every auth dependency under the router — none of which
|
||||
# run for a body that is already too large. The setup gate matters specifically: on an instance
|
||||
# that is not yet configured it opens a DB session per request, which is precisely the anonymous
|
||||
# pre-auth work SEC-20 exists to remove.
|
||||
app.add_middleware(bodylimit.BodyLimitMiddleware)
|
||||
|
||||
if settings.frontend_origin:
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[settings.frontend_origin],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def static_cache_headers(request, call_next):
|
||||
"""Long-cache the content-hashed SPA bundles. Vite hashes their filename, so a given
|
||||
@@ -183,6 +277,15 @@ async def security_headers(request, call_next):
|
||||
return response
|
||||
|
||||
|
||||
# The access log (app/logs.py): request id, route template, status, duration — one line, never a
|
||||
# query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole
|
||||
# request, including what every layer above does — the body cap included, so a 413 is still logged
|
||||
# with its request id and duration. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# which is why the account comes from `request.state.user_id` (set by the auth dependency, the one
|
||||
# place a session is actually resolved into an account) rather than from `request.session`.
|
||||
app.middleware("http")(log_requests)
|
||||
|
||||
|
||||
app.include_router(health.router)
|
||||
app.include_router(auth_routes.router)
|
||||
app.include_router(sync.router)
|
||||
@@ -191,6 +294,8 @@ app.include_router(feed.router)
|
||||
app.include_router(discover_routes.router)
|
||||
app.include_router(search_routes.router)
|
||||
app.include_router(me.router)
|
||||
app.include_router(diagnostics_routes.router)
|
||||
app.include_router(debug_sessions_routes.router)
|
||||
app.include_router(notifications.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(channels.router)
|
||||
@@ -204,6 +309,7 @@ app.include_router(admin.router)
|
||||
app.include_router(config_routes.router)
|
||||
app.include_router(scheduler_routes.router)
|
||||
app.include_router(audit_routes.router)
|
||||
app.include_router(admin_logs_routes.router)
|
||||
app.include_router(quota.router)
|
||||
app.include_router(version.router)
|
||||
app.include_router(setup_routes.router)
|
||||
|
||||
+149
-6
@@ -12,6 +12,8 @@ from sqlalchemy import (
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
LargeBinary,
|
||||
SmallInteger,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
@@ -21,9 +23,42 @@ from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
|
||||
from app.db import Base
|
||||
|
||||
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
||||
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
||||
# `User.display_name` (their Google profile name), `DownloadJob.display_name` and
|
||||
# `DownloadJob.display_uploader`. A value wider than the column is a psycopg DataError, which
|
||||
# `main.py` re-raises for everything but the NUL case — i.e. a bare 500 for a name somebody pasted.
|
||||
DISPLAY_TEXT_MAX = 255
|
||||
|
||||
|
||||
def display_text(raw: str | None) -> str | None:
|
||||
"""A user-supplied display string on its way into one of those columns: stripped, truncated,
|
||||
blank -> None.
|
||||
|
||||
One definition because there are SIX writers and they had drifted: `POST /api/downloads`,
|
||||
`POST /api/downloads/edit`, `POST /api/downloads/upload`, the metadata PATCH's two fields, and
|
||||
the Google sign-in callback. Only the PATCH truncated. Truncating is the right answer rather
|
||||
than refusing: the name is a label on work the user has already done — a file already
|
||||
transferred, an account already authenticated — and losing its tail is not worth failing the
|
||||
request over.
|
||||
|
||||
`test_upload.py` holds both halves: that every route assigning one of these fields goes through
|
||||
this helper (parsed, not grepped), and that each of those routes truncates rather than 500s. A
|
||||
sweep is exactly the kind of thing that gets done five-sixths of the way, which is how this
|
||||
ended up being written twice.
|
||||
"""
|
||||
return (raw or "").strip()[:DISPLAY_TEXT_MAX] or None
|
||||
|
||||
|
||||
from app.utils import canonical_email
|
||||
|
||||
|
||||
# The resume-position columns (VideoState / PlexState.position_seconds) are int4. The checkpoint
|
||||
# routes bound their bodies by this, so an out-of-range position is a 422 rather than a 500.
|
||||
POSITION_MAX_S = 2**31 - 1
|
||||
|
||||
|
||||
class TimestampMixin:
|
||||
"""Adds a server-defaulted `created_at`. Models that index created_at or order it
|
||||
differently (QuotaEvent, Notification, Message) keep their own column on purpose."""
|
||||
@@ -97,7 +132,17 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
# current_user rejects any cookie whose epoch is stale, so a stolen/copied cookie dies on those
|
||||
# events (client-side signed cookies otherwise stay valid until they expire).
|
||||
session_epoch: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
# When this account last signed in (Google or password), stamped by the one sign-in path,
|
||||
# routes/auth._establish_session. NULL = not since the column existed (0085; no backfill is
|
||||
# possible). The shared demo account is never stamped — its "last login" would mean nobody.
|
||||
last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The diagnostics level this user consented to (E22 S5) and when it lapses back to the base
|
||||
# level. Both NULL = base. Written only by `PUT /api/me/diagnostics`, which computes the expiry
|
||||
# itself; read through `app.diagnostics.effective_level`, never directly (an expired level is
|
||||
# not cleared eagerly — it simply stops counting).
|
||||
diagnostics_level: Mapped[str | None] = mapped_column(String(8))
|
||||
diagnostics_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
avatar_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
role: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
# The user's own YouTube channel id, resolved lazily (channels.list?mine=true) and
|
||||
@@ -225,6 +270,9 @@ class Channel(Base, TimestampMixin):
|
||||
details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# When the newest uploads were last read through the Data API because the RSS feed would not
|
||||
# answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock.
|
||||
feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
||||
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this channel only entered the catalog because a live YouTube search
|
||||
@@ -423,6 +471,7 @@ class VideoState(Base, UpdatedAtMixin):
|
||||
# status: a partially-watched video keeps status "new" but a non-zero position, so
|
||||
# it can render a progress bar and match the "in progress" feed filter. 0 = not
|
||||
# started / finished (trivially-early and near-finished positions are not stored).
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(
|
||||
Integer, default=0, server_default="0", nullable=False
|
||||
)
|
||||
@@ -533,6 +582,91 @@ class AuditLog(Base):
|
||||
)
|
||||
|
||||
|
||||
class LogEvent(Base):
|
||||
"""One stored log event (E22 S3) — the `log_events` table the diagnostics console reads. Rows are
|
||||
written by app/logsink.py's background writer, never inside a request's transaction, already
|
||||
redacted (the same `logs.record_fields` the stdout line is built from).
|
||||
|
||||
`user_id` has NO foreign key on purpose: the writer runs behind the event, so a line about an
|
||||
account can arrive after that account was purged, and a constraint would make it fail the whole
|
||||
batch it travels in. `auth.purge_user` deletes an account's rows explicitly instead, after
|
||||
`logs.forget_user` makes every later write in this process store such a line without the account;
|
||||
the retention job (`log_retention_days`) bounds whatever another process writes late."""
|
||||
|
||||
__tablename__ = "log_events"
|
||||
__table_args__ = (
|
||||
Index("ix_log_events_user_ts", "user_id", "ts"),
|
||||
# E16: an issue's bundle is `WHERE debug_session_id = …`; nearly every row has none.
|
||||
Index(
|
||||
"ix_log_events_debug_session",
|
||||
"debug_session_id",
|
||||
postgresql_where="debug_session_id IS NOT NULL",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
ts: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
level: Mapped[str] = mapped_column(String(10))
|
||||
sev: Mapped[int] = mapped_column(SmallInteger)
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
logger: Mapped[str] = mapped_column(String(128))
|
||||
event: Mapped[str | None] = mapped_column(String(64))
|
||||
msg: Mapped[str] = mapped_column(Text)
|
||||
attrs: Mapped[dict | None] = mapped_column(JSONB)
|
||||
exc: Mapped[str | None] = mapped_column(Text)
|
||||
request_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[int | None] = mapped_column(Integer)
|
||||
debug_session_id: Mapped[str | None] = mapped_column(String(64))
|
||||
job_id: Mapped[int | None] = mapped_column(Integer)
|
||||
version: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class DebugSession(Base):
|
||||
"""One recording: Start → reproduce → Stop (E16, app/debugsession.py). While it is open, every
|
||||
request of its user that names it (`X-Debug-Session`) is logged at `level` and tagged with `id`,
|
||||
and the issue it ends in is built from the rows carrying that tag. `until` is the hard cap the
|
||||
server set at Start; an open session past it is over whether or not the sweep has closed it yet.
|
||||
One OPEN session per user (`uq_debug_sessions_open_per_user`, partial on `ended_at IS NULL`)."""
|
||||
|
||||
__tablename__ = "debug_sessions"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"uq_debug_sessions_open_per_user",
|
||||
"user_id",
|
||||
unique=True,
|
||||
postgresql_where="ended_at IS NULL",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
level: Mapped[str] = mapped_column(String(8))
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
until: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# app/debugsession.py `EndReason`: stopped by the user, or past `until`.
|
||||
ended_reason: Mapped[str | None] = mapped_column(String(16))
|
||||
issue_url: Mapped[str | None] = mapped_column(String(512))
|
||||
# The issue being written from it (E16 S3; app/debugsession.py `save_draft` holds its shape).
|
||||
draft: Mapped[dict | None] = mapped_column(JSONB)
|
||||
|
||||
|
||||
class DebugScreenshot(Base):
|
||||
"""A screenshot attached to a debug session (E16) — pasted, dropped, or captured from the pill.
|
||||
Stored in the row: bounded per session (app/debugsession.py), and gone with it by cascade."""
|
||||
|
||||
__tablename__ = "debug_screenshots"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
session_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("debug_sessions.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
mime: Mapped[str] = mapped_column(String(16))
|
||||
size: Mapped[int] = mapped_column(Integer)
|
||||
data: Mapped[bytes] = mapped_column(LargeBinary)
|
||||
|
||||
|
||||
class AppState(Base):
|
||||
"""Single-row global app state (admin-controlled)."""
|
||||
|
||||
@@ -847,11 +981,11 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
ForeignKey("download_profiles.id", ondelete="SET NULL")
|
||||
)
|
||||
profile_snapshot: Mapped[dict] = mapped_column(JSON)
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
display_name: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
# Per-user display overrides + extras (like display_name — metadata only, never the file).
|
||||
# `display_uploader`/`display_uploader_url` override the asset's auto channel name/link;
|
||||
# `extra_links` is an optional list of extra reference URLs shown on the card + watch page.
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(255))
|
||||
display_uploader: Mapped[str | None] = mapped_column(String(DISPLAY_TEXT_MAX))
|
||||
display_uploader_url: Mapped[str | None] = mapped_column(Text)
|
||||
extra_links: Mapped[list | None] = mapped_column(JSON)
|
||||
status: Mapped[str] = mapped_column(
|
||||
@@ -865,6 +999,8 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
error_code: Mapped[str | None] = mapped_column(
|
||||
String(24)
|
||||
) # category, see downloads.errors.classify
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
@@ -1088,6 +1224,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
||||
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline (E30)
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
@@ -1249,6 +1386,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
||||
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline (E30)
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
@@ -1279,7 +1417,8 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
||||
top few cast. Rows whose name no longer appears in ANY item's cast_names/directors (a library
|
||||
removal orphaned them) are pruned by the maintenance sweep, mirroring how Plex drops orphaned
|
||||
metadata. The filterable cast lives in plex_items/plex_shows.cast_names (self-cleaning with the
|
||||
row); this table is ONLY the headshot cache."""
|
||||
row); this table is the people's own facts: the headshot link, the TMDB link and the TMDB bio
|
||||
(the headshot FILE lives in the image cache, filled by the people_enrich job)."""
|
||||
|
||||
__tablename__ = "plex_people"
|
||||
|
||||
@@ -1289,11 +1428,15 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# TMDB person id, linked by name from a title's TMDB credits (E5 S2 lays the foundation; the /person
|
||||
# biography fetch + person-page render land in S6). NULL until a TMDB-enriched title names this person.
|
||||
tmdb_person_id: Mapped[int | None] = mapped_column(Integer)
|
||||
# --- TMDB person enrichment (E5 S6): fetched from /person/{tmdb_person_id} the first time the person
|
||||
# page is opened, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||
# --- TMDB person enrichment (E5 S6): fetched from /person/{tmdb_person_id} by the people_enrich job
|
||||
# (E30 S6), or by the first person-page open the job hasn't reached, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||
# no-op fetch so it isn't retried every open). `known_for` = top combined-credits [{tmdb_id,
|
||||
# media_type, title}], resolved to library rows by guid at serve time (like recommendations). ---
|
||||
biography: Mapped[str | None] = mapped_column(Text)
|
||||
# TMDB's Hungarian biography (E30 S4), from the same request's `translations`; a Hungarian reader is
|
||||
# served it, and the instance-language `biography` when TMDB has none. Birthplace, aliases and
|
||||
# keywords have no Hungarian on TMDB, so they stay in the instance language.
|
||||
biography_hu: Mapped[str | None] = mapped_column(Text)
|
||||
birthday: Mapped[str | None] = mapped_column(String(10)) # "YYYY-MM-DD", served as-is
|
||||
deathday: Mapped[str | None] = mapped_column(String(10))
|
||||
place_of_birth: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
+32
-14
@@ -15,20 +15,35 @@ def _art_url(rating_key: str, art_key: str | None) -> str | None:
|
||||
|
||||
|
||||
# Card hover-overview: a short synopsis for the reveal panel, trimmed so a strip/related row of cards
|
||||
# stays a few hundred bytes each rather than carrying full synopses. (These non-grid card paths serve
|
||||
# the EN summary/rating; the main grid coalesces the HU twin in browse.unified_library.)
|
||||
# stays a few hundred bytes each rather than carrying full synopses.
|
||||
def _card_summary(summary: str | None) -> str | None:
|
||||
return (summary or "")[:300] or None
|
||||
|
||||
|
||||
def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
||||
def norm_lang(lang: str | None) -> str:
|
||||
"""The overlay has an EN base + an optional HU twin; anything other than `hu` serves EN."""
|
||||
return "hu" if lang == "hu" else "en"
|
||||
|
||||
|
||||
# Every card speaks the reader's language (E30): `lang="hu"` serves the HU overlay twin's title and
|
||||
# summary, falling back to EN where a row has none — the same rule as browse.unified_library's
|
||||
# `title_col`/`summary_col`. These rails once served English to a Hungarian reader.
|
||||
def display_title(row: PlexItem | PlexShow, lang: str) -> str:
|
||||
return (row.title_hu or row.title) if lang == "hu" else row.title
|
||||
|
||||
|
||||
def display_summary(row: PlexItem | PlexShow, lang: str) -> str | None:
|
||||
return (row.summary_hu or row.summary) if lang == "hu" else row.summary
|
||||
|
||||
|
||||
def movie_card(it: PlexItem, st: PlexState | None, *, lang: str) -> dict:
|
||||
return {
|
||||
"id": it.rating_key,
|
||||
"type": "movie",
|
||||
"title": it.title,
|
||||
"title": display_title(it, lang),
|
||||
"year": it.year,
|
||||
"rating": it.rating,
|
||||
"summary": _card_summary(it.summary),
|
||||
"summary": _card_summary(display_summary(it, lang)),
|
||||
"duration_seconds": it.duration_s,
|
||||
"thumb": f"/api/plex/image/{it.rating_key}",
|
||||
"art": _art_url(it.rating_key, it.art_key),
|
||||
@@ -39,14 +54,14 @@ def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def show_card(sh: PlexShow, status: str = "new") -> dict:
|
||||
def show_card(sh: PlexShow, status: str = "new", *, lang: str) -> dict:
|
||||
return {
|
||||
"id": sh.rating_key,
|
||||
"type": "show",
|
||||
"title": sh.title,
|
||||
"title": display_title(sh, lang),
|
||||
"year": sh.year,
|
||||
"rating": sh.rating,
|
||||
"summary": _card_summary(sh.summary),
|
||||
"summary": _card_summary(display_summary(sh, lang)),
|
||||
"thumb": f"/api/plex/image/{sh.rating_key}",
|
||||
"art": _art_url(sh.rating_key, sh.art_key),
|
||||
"season_count": sh.child_count,
|
||||
@@ -55,12 +70,12 @@ def show_card(sh: PlexShow, status: str = "new") -> dict:
|
||||
}
|
||||
|
||||
|
||||
def episode_card(e: PlexItem, st: PlexState | None) -> dict:
|
||||
def episode_card(e: PlexItem, st: PlexState | None, *, lang: str) -> dict:
|
||||
return {
|
||||
"id": e.rating_key,
|
||||
"type": "episode",
|
||||
"title": e.title,
|
||||
"summary": e.summary,
|
||||
"title": display_title(e, lang),
|
||||
"summary": display_summary(e, lang),
|
||||
"season_number": e.season_number,
|
||||
"episode_number": e.episode_number,
|
||||
"duration_seconds": e.duration_s,
|
||||
@@ -77,12 +92,15 @@ def leaf_card(
|
||||
st: PlexState | None,
|
||||
show_title: str | None = None,
|
||||
show_rk: str | None = None,
|
||||
*,
|
||||
lang: str,
|
||||
) -> dict:
|
||||
"""A playable-leaf card (movie or episode). Used by playlists, which can mix both kinds.
|
||||
`show_rk` (the show's rating_key) lets the client group a playlist's episodes by show/season."""
|
||||
`show_rk` (the show's rating_key) lets the client group a playlist's episodes by show/season.
|
||||
`show_title` is the caller's, already in `lang` (see `display_title`)."""
|
||||
if it.kind == "episode":
|
||||
card = episode_card(it, st)
|
||||
card = episode_card(it, st, lang=lang)
|
||||
card["show_title"] = show_title
|
||||
card["show_id"] = show_rk
|
||||
return card
|
||||
return movie_card(it, st)
|
||||
return movie_card(it, st, lang=lang)
|
||||
|
||||
@@ -6,16 +6,15 @@ metadata, image proxying, and (P5) optional watch-state write-back. Requests JSO
|
||||
``Accept: application/json`` header (Plex serves XML by default).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class PlexError(Exception):
|
||||
@@ -106,7 +105,18 @@ class PlexClient:
|
||||
Best-effort (needs the library's related data) — returns [] if Plex has none."""
|
||||
try:
|
||||
mc = self._get(f"/library/metadata/{rating_key}/related")
|
||||
except PlexError:
|
||||
except PlexError as exc:
|
||||
# An empty list is also the answer for an item that genuinely has no related titles,
|
||||
# so the caller cannot tell the two apart — and it deliberately declines to CACHE an
|
||||
# empty one for exactly that reason. This line is what says which it was.
|
||||
log.debug(
|
||||
"the related items for a Plex item could not be read",
|
||||
extra=logs.event(
|
||||
"plex.client.related_unavailable",
|
||||
rating_key=rating_key,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for hub in mc.get("Hub", []) or []:
|
||||
@@ -139,6 +149,8 @@ class PlexClient:
|
||||
raise PlexError(str(e)) from e
|
||||
try:
|
||||
return (r.json() or {}).get("MediaContainer", {}) or {} if r.content else {}
|
||||
# silent-ok: the HTTP call already succeeded — this is a write endpoint, and Plex answers
|
||||
# those with an empty or non-JSON body as a matter of course.
|
||||
except ValueError:
|
||||
return {}
|
||||
|
||||
@@ -197,7 +209,12 @@ class PlexClient:
|
||||
)
|
||||
|
||||
def unscrobble(self, rating_key: str) -> None:
|
||||
"""Mark the item *unwatched* on the token's Plex account (clears viewCount)."""
|
||||
"""Mark the item *unwatched* on the token's Plex account.
|
||||
|
||||
Clears the view count AND the resume point (`viewOffset`) — measured against the live server
|
||||
on 2026-09-09, from both a watched+offset state and an offset-only one. This one call is the
|
||||
whole un-mark; `/:/timeline?time=0` and `/:/progress?time=0` were both tried there and do
|
||||
NOT clear an offset, so do not reach for them."""
|
||||
self._write(
|
||||
"GET",
|
||||
f"/:/unscrobble?identifier={self._LIBRARY_IDENTIFIER}"
|
||||
|
||||
@@ -22,6 +22,8 @@ def rich_detail(meta: dict) -> dict:
|
||||
if str(r.get("image") or "").startswith("imdb://"):
|
||||
try:
|
||||
imdb_rating = round(float(r.get("value")), 1)
|
||||
# silent-ok: no IMDb score, which is the common and legitimate case — the UI has a
|
||||
# separate audience score to fall back on and shows nothing if neither exists.
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
break
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""One Hungarian word per Plex genre (E30).
|
||||
|
||||
Plex's Hungarian agent returns a MIXED vocabulary: some genres come back translated ("Dráma",
|
||||
"Vígjáték"), others stay English ("Action", "Adventure", "Biography"), and a title with no HU twin has
|
||||
only its English genres. The HU genre list therefore offered "Drama" beside "Dráma" and a dozen English
|
||||
words. This map sends every genre word we have met — English (Plex EN agent, TMDB, IMDb) or Plex's
|
||||
Hungarian — to the ONE word a Hungarian reader sees. The overlay sync writes it into `genres_hu`, and a
|
||||
genre filter is read through it too, so a saved filter still holding "Action" keeps matching "Akció".
|
||||
|
||||
A word not in the map passes through unchanged: an unknown genre stays visible (in whatever language
|
||||
Plex gave it) rather than disappearing.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
# Keys are matched case-insensitively (see `_BY_KEY`). The Hungarian values double as keys mapping to
|
||||
# themselves, so running a list through twice changes nothing.
|
||||
_HU: dict[str, str] = {
|
||||
"Action": "Akció",
|
||||
"Action & Adventure": "Akció és kaland",
|
||||
"Adventure": "Kaland",
|
||||
"Animation": "Animációs",
|
||||
"Anime": "Anime",
|
||||
"Biography": "Életrajzi",
|
||||
"Children": "Gyerek",
|
||||
"Comedy": "Vígjáték",
|
||||
"Crime": "Bűnügyi",
|
||||
"Documentary": "Dokumentum",
|
||||
"Drama": "Dráma",
|
||||
"Family": "Családi",
|
||||
"Fantasy": "Fantasy",
|
||||
"Film-Noir": "Film noir",
|
||||
"Food": "Gasztronómia",
|
||||
"Game Show": "Vetélkedő",
|
||||
"Game-Show": "Vetélkedő",
|
||||
"History": "Történelmi",
|
||||
"Home and Garden": "Otthon és kert",
|
||||
"Horror": "Horror",
|
||||
"Indie": "Független",
|
||||
"Kids": "Gyerek",
|
||||
"Martial Arts": "Harcművészet",
|
||||
"Mini-Series": "Minisorozat",
|
||||
"Music": "Zenei",
|
||||
"Musical": "Musical",
|
||||
"Mystery": "Rejtély",
|
||||
"News": "Hírek",
|
||||
"Reality": "Valóságshow",
|
||||
"Reality-TV": "Valóságshow",
|
||||
"Romance": "Romantikus",
|
||||
"Sci-Fi": "Sci-Fi",
|
||||
"Sci-Fi & Fantasy": "Sci-Fi és fantasy",
|
||||
"Science Fiction": "Sci-Fi",
|
||||
"Short": "Rövidfilm",
|
||||
"Soap": "Szappanopera",
|
||||
"Sport": "Sport",
|
||||
"Suspense": "Feszültség",
|
||||
"Talk": "Talkshow",
|
||||
"Talk Show": "Talkshow",
|
||||
"Thriller": "Thriller",
|
||||
"Travel": "Utazás",
|
||||
"TV Movie": "TV film",
|
||||
"War": "Háborús",
|
||||
"War & Politics": "Háború és politika",
|
||||
"Western": "Western",
|
||||
# Plex's own Hungarian words, pinned to themselves (and to the spelling above where they differ).
|
||||
"Akció": "Akció",
|
||||
"Animációs": "Animációs",
|
||||
"Bűnügyi": "Bűnügyi",
|
||||
"Családi": "Családi",
|
||||
"Dokumentum": "Dokumentum",
|
||||
"Dokumentumfilm": "Dokumentum",
|
||||
"Dráma": "Dráma",
|
||||
"Életrajzi": "Életrajzi",
|
||||
"Háborús": "Háborús",
|
||||
"Kaland": "Kaland",
|
||||
"Rejtély": "Rejtély",
|
||||
"Romantikus": "Romantikus",
|
||||
"Történelmi": "Történelmi",
|
||||
"TV film": "TV film",
|
||||
"Vígjáték": "Vígjáték",
|
||||
"Zene": "Zenei",
|
||||
"Zenei": "Zenei",
|
||||
}
|
||||
_BY_KEY = {k.casefold(): v for k, v in _HU.items()}
|
||||
|
||||
|
||||
def hu_genre(word: str) -> str:
|
||||
"""The Hungarian word for one genre; an unknown word comes back unchanged."""
|
||||
return _BY_KEY.get(word.strip().casefold(), word)
|
||||
|
||||
|
||||
def hu_genres(words: Iterable[str] | None) -> list[str] | None:
|
||||
"""A genre list in Hungarian, in its first-seen order, each word once ("Drama" + "Dráma" → one
|
||||
"Dráma"). None or empty stays None, like the listing's own `_tags`."""
|
||||
out: list[str] = []
|
||||
for w in words or ():
|
||||
hu = hu_genre(w)
|
||||
if hu not in out:
|
||||
out.append(hu)
|
||||
return out or None
|
||||
@@ -8,13 +8,19 @@ few hundred MB at most for a big library — acceptable, matching the download d
|
||||
The cache directory (``cache.CACHE_DIR``) is shared with the subtitle cache (see ``subtitles.py``).
|
||||
"""
|
||||
|
||||
import time
|
||||
from datetime import timedelta
|
||||
|
||||
from fastapi import Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import PlexCollection, PlexItem, PlexSeason, PlexShow
|
||||
from app.plex.cache import CACHE_DIR
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
_EXT_CT = [("jpg", "image/jpeg"), ("png", "image/png"), ("webp", "image/webp")]
|
||||
_CT_EXT = {ct: ext for ext, ct in _EXT_CT}
|
||||
# Target box per variant — Plex resizes to these server-side. Posters fill ≤176px cells (grid/info/
|
||||
@@ -46,11 +52,56 @@ def read(key: str) -> tuple[bytes, str] | None:
|
||||
try:
|
||||
if p.exists() and p.stat().st_size > 0:
|
||||
return p.read_bytes(), ct
|
||||
# silent-ok: a miss. The caller fetches from Plex and writes the file again, and the write
|
||||
# path below is the one that reports a cache that cannot be used.
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def has(key: str) -> bool:
|
||||
"""A cached, non-empty file exists for `key` (a prefetch's cheap check — no read)."""
|
||||
for ext, _ in _EXT_CT:
|
||||
try:
|
||||
if (CACHE_DIR / f"{key}.{ext}").stat().st_size > 0:
|
||||
return True
|
||||
# silent-ok: a miss, as in read().
|
||||
except OSError:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def drop(key: str) -> None:
|
||||
"""Remove `key`'s file, whatever its extension (a superseded variant). Best-effort."""
|
||||
for ext, _ in _EXT_CT:
|
||||
try:
|
||||
(CACHE_DIR / f"{key}.{ext}").unlink(missing_ok=True)
|
||||
# silent-ok: a leftover original only costs disk; the small twin is already written.
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def mark_miss(key: str) -> None:
|
||||
"""Remember that `key`'s source could not be fetched (an empty `.miss` file; its mtime is when), so
|
||||
a prefetch can leave a dead source alone for a while. Best-effort."""
|
||||
try:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(CACHE_DIR / f"{key}.miss").touch()
|
||||
# silent-ok: without the mark the dead source is only asked for again on the next run.
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def missed_within(key: str, age: timedelta) -> bool:
|
||||
"""`key` was marked as a miss less than `age` ago."""
|
||||
try:
|
||||
mtime = (CACHE_DIR / f"{key}.miss").stat().st_mtime
|
||||
# silent-ok: no mark (the usual case).
|
||||
except OSError:
|
||||
return False
|
||||
return time.time() - mtime < age.total_seconds()
|
||||
|
||||
|
||||
def write(key: str, data: bytes, ct: str) -> None:
|
||||
ext = _CT_EXT.get((ct or "").split(";")[0].strip(), "jpg")
|
||||
try:
|
||||
@@ -58,8 +109,14 @@ def write(key: str, data: bytes, ct: str) -> None:
|
||||
tmp = CACHE_DIR / f"{key}.{ext}.tmp"
|
||||
tmp.write_bytes(data)
|
||||
tmp.replace(CACHE_DIR / f"{key}.{ext}") # atomic publish
|
||||
except OSError:
|
||||
pass # cache is best-effort
|
||||
except OSError as exc:
|
||||
# Best-effort, and silently so: with an unwritable cache every poster on every grid goes
|
||||
# to Plex live, for as long as nobody notices. Debug, because a grid is ~100 images and
|
||||
# the point is the condition, not the count.
|
||||
log.debug(
|
||||
"the Plex image cache could not be written",
|
||||
extra=logs.event("plex.imgcache.write_failed", reason=type(exc).__name__),
|
||||
)
|
||||
|
||||
|
||||
def response(data: bytes, ct: str) -> Response:
|
||||
|
||||
@@ -11,5 +11,11 @@ _M = TypeVar("_M")
|
||||
def by_rating_key(db: Session, model: type[_M], rating_key: str) -> _M | None:
|
||||
"""The mirrored row for a Plex rating_key, or None. `model` is any of the rating_key-keyed Plex
|
||||
tables (PlexItem / PlexShow / PlexSeason / PlexCollection); the return type is that model, so call
|
||||
sites keep their `Model | None` static type."""
|
||||
sites keep their `Model | None` static type.
|
||||
|
||||
A key with a NUL in it (`%00` in a path segment) names no row — a 404, like any unknown key.
|
||||
Postgres refuses NUL in text, so asking would be a DataError; the app-wide handler turns the
|
||||
places that DO ask (lists and filters compared to rating_key) into a 422 (see main.py)."""
|
||||
if "\x00" in str(rating_key):
|
||||
return None
|
||||
return db.execute(select(model).where(model.rating_key == str(rating_key))).scalars().first()
|
||||
|
||||
@@ -12,7 +12,9 @@ from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def parse_map(raw: str) -> list[tuple[str, str]]:
|
||||
@@ -47,6 +49,13 @@ def local_media_path(db: Session, plex_path: str | None) -> Path | None:
|
||||
return None
|
||||
try:
|
||||
p = Path(map_path(db, plex_path)).resolve()
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# None is answered as a 404 "no local file", which is also the answer for a title the
|
||||
# server does not hold — so a dead mount or a wrong path map reads as missing media for
|
||||
# the whole library at once. The path itself stays out: it carries the title.
|
||||
log.warning(
|
||||
"a Plex media path could not be resolved locally",
|
||||
extra=logs.event("plex.paths.resolve_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
return p if p.is_file() else None
|
||||
|
||||
+176
-10
@@ -3,13 +3,23 @@ or show (cast_names + the people_text search blob + the plex_people headshot cac
|
||||
orphaned people. Used by BOTH the info-page lazy enrichment (routes.plex) and the background rich
|
||||
re-sync (plex.sync), so the two paths never diverge."""
|
||||
|
||||
from urllib.parse import quote
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import timedelta
|
||||
from typing import Literal
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexPerson
|
||||
from app.plex import imgcache
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Cast/crew photos come from Plex's PUBLIC metadata CDN (no token). We proxy them (host-whitelisted)
|
||||
# so the user's browser makes no third-party request (privacy + self-host CSP friendliness).
|
||||
@@ -19,9 +29,7 @@ PERSON_IMG_HOST = "https://metadata-static.plex.tv/"
|
||||
def person_thumb_url(thumb: str | None) -> str | None:
|
||||
"""Proxied /person-image URL for a Plex CDN headshot, or None when it's missing / off-CDN."""
|
||||
t = str(thumb or "")
|
||||
return (
|
||||
f"/api/plex/person-image?u={quote(t, safe='')}" if t.startswith(PERSON_IMG_HOST) else None
|
||||
)
|
||||
return f"/api/plex/person-image?u={quote(t, safe='')}" if is_portrait_url(t) else None
|
||||
|
||||
|
||||
def cast_from_meta(meta: dict, limit: int = 24) -> list[dict]:
|
||||
@@ -89,14 +97,23 @@ def enrich_row_people(db: Session, row, cast: list[dict]) -> bool:
|
||||
return dirty
|
||||
|
||||
|
||||
# Every credit reference: one person name per (movie/show, credit column). The ONE list of credit
|
||||
# columns, so the orphan prune and the enrichment order never disagree about who is credited.
|
||||
_CREDIT_COLUMNS = [
|
||||
(table, col) for table in ("plex_items", "plex_shows") for col in ("cast_names", "directors")
|
||||
]
|
||||
|
||||
|
||||
def _credit_refs_sql(union: Literal["UNION", "UNION ALL"]) -> str:
|
||||
return f" {union} ".join(
|
||||
f"SELECT jsonb_array_elements_text({col}) FROM {table} WHERE jsonb_typeof({col}) = 'array'"
|
||||
for table, col in _CREDIT_COLUMNS
|
||||
)
|
||||
|
||||
|
||||
# The person names still referenced by ANY movie/show (cast OR directors). plex_people rows outside
|
||||
# this set are orphans left by a library removal.
|
||||
_REFERENCED_PEOPLE_SQL = """
|
||||
SELECT jsonb_array_elements_text(cast_names) FROM plex_items WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_items WHERE jsonb_typeof(directors) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(cast_names) FROM plex_shows WHERE jsonb_typeof(cast_names) = 'array'
|
||||
UNION SELECT jsonb_array_elements_text(directors) FROM plex_shows WHERE jsonb_typeof(directors) = 'array'
|
||||
"""
|
||||
_REFERENCED_PEOPLE_SQL = _credit_refs_sql("UNION")
|
||||
|
||||
|
||||
def prune_orphan_people(db: Session) -> int:
|
||||
@@ -113,3 +130,152 @@ def prune_orphan_people(db: Session) -> int:
|
||||
)
|
||||
db.commit()
|
||||
return res.rowcount or 0
|
||||
|
||||
|
||||
# How often each person is credited across movies, shows and episodes (cast OR director) — the order the
|
||||
# people_enrich job works in, so the faces met most often are local first.
|
||||
_CREDIT_COUNTS_SQL = (
|
||||
f"SELECT ref, count(*) AS n FROM ({_credit_refs_sql('UNION ALL')}) AS refs(ref) GROUP BY ref"
|
||||
)
|
||||
_NEEDS = {
|
||||
# linked to TMDB, bio never fetched
|
||||
"bio": "p.tmdb_person_id IS NOT NULL AND p.tmdb_enriched_at IS NULL",
|
||||
# has a Plex headshot (whether it is cached is the disk's answer, not the row's)
|
||||
"portrait": "p.photo_thumb IS NOT NULL",
|
||||
}
|
||||
|
||||
|
||||
def people_by_credits(
|
||||
db: Session, need: Literal["bio", "portrait"], limit: int | None
|
||||
) -> list[int]:
|
||||
"""plex_people ids that `need` something, the most-credited first (ties by id)."""
|
||||
rows = db.execute(
|
||||
text(
|
||||
f"SELECT p.id FROM plex_people p LEFT JOIN ({_CREDIT_COUNTS_SQL}) c ON c.ref = p.name "
|
||||
f"WHERE {_NEEDS[need]} ORDER BY coalesce(c.n, 0) DESC, p.id"
|
||||
+ (" LIMIT :limit" if limit is not None else "")
|
||||
),
|
||||
{"limit": limit} if limit is not None else {},
|
||||
)
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
# --- Portraits (E30 S6) ---------------------------------------------------------------------------
|
||||
# A Plex CDN headshot is a full-size original (~170 KB on average, measured). Plex's photo transcoder
|
||||
# fits it into this box (~13 KB), which covers the largest place a portrait is drawn (the person page's
|
||||
# 6rem avatar) at 2x density and a raised text size. The box is part of the cache key, like a poster's:
|
||||
# a new box re-caches instead of serving a stale size. Only a resized image is ever written under it —
|
||||
# an original fetched without Plex keeps its own key, and goes once its small twin lands.
|
||||
PORTRAIT_BOX = 320
|
||||
# A headshot Plex answered "not found" for (a dead CDN file) is not asked for again this long, so a
|
||||
# few dead faces never hold up the rest of the prefetch.
|
||||
PORTRAIT_RETRY = timedelta(days=7)
|
||||
# The one shape a Plex CDN headshot has (…/7/people/<hash>.jpg): plain path segments, no query, no
|
||||
# fragment, no dot segment or empty segment (an HTTP client folds `a/../b` and the CDN `a//b` onto one
|
||||
# file). The person-image proxy takes nothing else, so it cannot be made to cache endless spellings of
|
||||
# one file (the image cache has no eviction). Used where a stored headshot is made, served and warmed.
|
||||
_SEGMENT = r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*" # a name, dots only between its parts
|
||||
_PORTRAIT_URL = re.compile(re.escape(PERSON_IMG_HOST) + _SEGMENT + rf"(?:/{_SEGMENT})*")
|
||||
|
||||
|
||||
def is_portrait_url(url: str) -> bool:
|
||||
return _PORTRAIT_URL.fullmatch(url) is not None
|
||||
|
||||
|
||||
def _portrait_keys(url: str) -> tuple[str, str]:
|
||||
"""(the resized portrait's key, the full-size original's key)."""
|
||||
base = "person_" + hashlib.sha1(url.encode()).hexdigest()
|
||||
return f"{base}_{PORTRAIT_BOX}", base
|
||||
|
||||
|
||||
def portrait_cached(url: str) -> tuple[bytes, str] | None:
|
||||
"""The cached portrait: the resized one, else an original kept from a time without Plex."""
|
||||
small, original = _portrait_keys(url)
|
||||
return imgcache.read(small) or imgcache.read(original)
|
||||
|
||||
|
||||
def cache_portrait(plex: PlexClient | None, url: str) -> tuple[bytes, str]:
|
||||
"""Fetch one CDN headshot and cache it. With a Plex client it is resized (the box key; the original
|
||||
is removed); without one the original comes straight from the public CDN and is cached under its own
|
||||
key, so the job still resizes it once Plex is there. Raises PlexError / httpx.HTTPError."""
|
||||
small, original = _portrait_keys(url)
|
||||
if plex is None:
|
||||
with httpx.Client(timeout=10.0, follow_redirects=True) as c:
|
||||
r = c.get(url)
|
||||
r.raise_for_status()
|
||||
data, ct = r.content, r.headers.get("content-type", "image/jpeg")
|
||||
imgcache.write(original, data, ct)
|
||||
return data, ct
|
||||
data, ct = plex.image_bytes(url, PORTRAIT_BOX, PORTRAIT_BOX)
|
||||
imgcache.write(small, data, ct)
|
||||
imgcache.drop(original)
|
||||
return data, ct
|
||||
|
||||
|
||||
def cdn_url(photo_thumb: str) -> str | None:
|
||||
"""The CDN url inside a stored proxied `/person-image?u=` value (see person_thumb_url)."""
|
||||
_, _, q = photo_thumb.partition("?u=")
|
||||
url = unquote(q)
|
||||
return url if is_portrait_url(url) else None
|
||||
|
||||
|
||||
# Plex failing this many headshots in a row ends a run: it is Plex that is failing, not the files.
|
||||
_PLEX_DOWN_AFTER = 3
|
||||
# The statuses that mean the one file is gone.
|
||||
_DEAD = frozenset({404, 410})
|
||||
|
||||
|
||||
def warm_portraits(db: Session, limit: int) -> dict:
|
||||
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first (E30 S6), so a
|
||||
person's face comes from local disk. Needs Plex (the resize) and the Plex module on; without them
|
||||
nothing is prefetched — full-size originals for every person would cost gigabytes. A headshot Plex
|
||||
answers "not found" for is marked and left for PORTRAIT_RETRY; Plex failing in a row ends the run."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
return {"cached": 0, "skipped": "disabled"}
|
||||
try:
|
||||
plex = PlexClient(db)
|
||||
# silent-ok: no Plex server set up is a setting, not a failure; the run's result says "disabled".
|
||||
except PlexNotConfigured:
|
||||
return {"cached": 0, "skipped": "disabled"}
|
||||
photos: dict[int, str] = {
|
||||
pid: thumb
|
||||
for pid, thumb in db.execute(
|
||||
select(PlexPerson.id, PlexPerson.photo_thumb).where(PlexPerson.photo_thumb.is_not(None))
|
||||
).tuples()
|
||||
if thumb
|
||||
}
|
||||
# `down` counts Plex failing in a row (any answer from Plex breaks it), `plex_failed` in all
|
||||
done = dead = plex_failed = down = 0
|
||||
with plex:
|
||||
for pid in people_by_credits(db, "portrait", None):
|
||||
if done + dead >= limit or down >= _PLEX_DOWN_AFTER:
|
||||
break
|
||||
url = cdn_url(photos.get(pid) or "")
|
||||
if url is None:
|
||||
continue
|
||||
key = _portrait_keys(url)[0]
|
||||
if imgcache.has(key) or imgcache.missed_within(key, PORTRAIT_RETRY):
|
||||
continue
|
||||
try:
|
||||
cache_portrait(plex, url)
|
||||
done, down = done + 1, 0
|
||||
except PlexError as exc:
|
||||
# "Not found" is Plex's answer about this one file (a dead CDN headshot: measured 404,
|
||||
# the CDN itself says 403). Anything else — no answer, a refused token, a 5xx — is
|
||||
# Plex failing, which must neither mark the file nor keep the run going.
|
||||
cause = exc.__cause__
|
||||
if isinstance(cause, httpx.HTTPStatusError) and cause.response.status_code in _DEAD:
|
||||
dead, down = dead + 1, 0 # Plex answered: it is up, the streak is broken
|
||||
imgcache.mark_miss(key)
|
||||
else:
|
||||
plex_failed, down = plex_failed + 1, down + 1
|
||||
log.warning(
|
||||
"a portrait could not be cached",
|
||||
extra=logs.event(
|
||||
"plex.people.portrait_failed",
|
||||
person_id=pid,
|
||||
reason=type(exc.__cause__).__name__,
|
||||
),
|
||||
)
|
||||
failed = dead + plex_failed
|
||||
return {"cached": done, **({"failed": failed} if failed else {})}
|
||||
|
||||
@@ -11,9 +11,26 @@ from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import PlexItem, PlexLibrary
|
||||
from app.plex.genres import hu_genre
|
||||
|
||||
TS_CONFIG = "public.unaccent_simple"
|
||||
|
||||
# The ONE Plex sort vocabulary, shared by every surface that offers the Sort control (the browse grid
|
||||
# and the collection info page) and mirrored by the frontend's `plexSort.ts`. It lives here rather than
|
||||
# inside a router because the two surfaces order DIFFERENT things — browse orders a union subquery in
|
||||
# SQL, a collection orders its already-loaded member list in Python — and a key that one of them cannot
|
||||
# honour would put an option in the UI that silently does nothing. Tests pin both implementations to
|
||||
# this tuple, so adding a key here fails the gate until every surface can order by it.
|
||||
PLEX_SORT_KEYS: tuple[str, ...] = (
|
||||
"added",
|
||||
"lastwatched",
|
||||
"release",
|
||||
"year",
|
||||
"rating",
|
||||
"duration",
|
||||
"title",
|
||||
)
|
||||
|
||||
# --- Language-aware column pickers (E5 S6) ----------------------------------
|
||||
#
|
||||
# The HU-overlay twin (E5 S5) stores Hungarian metadata in `*_hu` columns on the SAME EN row. When
|
||||
@@ -167,7 +184,13 @@ def meta_filter_conds(model, p: dict, lang: str = "en") -> list:
|
||||
gsel = split_csv(p.get("genres"))
|
||||
if gsel:
|
||||
gcol = genres_col(model, lang)
|
||||
gc = [gcol.contains([g]) for g in gsel]
|
||||
if lang == "hu":
|
||||
# The HU column holds one Hungarian word per genre (plex/genres.py), so a filter saved while
|
||||
# it offered "Action" is read as "Akció" too. The word as picked still matches as well: a row
|
||||
# the sync has not reached since keeps its English (or Plex's mixed) list until it does.
|
||||
gc = [or_(*(gcol.contains([w]) for w in dict.fromkeys((hu_genre(g), g)))) for g in gsel]
|
||||
else:
|
||||
gc = [gcol.contains([g]) for g in gsel]
|
||||
conds.append(and_(*gc) if p.get("genre_mode") == "all" else or_(*gc))
|
||||
crs = split_csv(p.get("content_ratings"))
|
||||
if crs:
|
||||
@@ -282,10 +305,6 @@ class LibraryFilters:
|
||||
}
|
||||
|
||||
|
||||
def like_escape(s: str) -> str:
|
||||
return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
# --- Watch-state scope (shared by the /library grid AND the /facets aggregation) ---------------
|
||||
#
|
||||
# The `show` scope (all | watched | in_progress | unwatched) filters the browse result by per-user
|
||||
|
||||
@@ -43,7 +43,9 @@ def rollup(cards: list[dict]) -> dict:
|
||||
]
|
||||
resume = None
|
||||
if inprog:
|
||||
resume = inprog[-1] # continue the latest-started episode
|
||||
# The LAST in-progress episode in broadcast order — no timestamp is consulted. Home's
|
||||
# `discover.queries.next_episode` picks the same one, so both Resume buttons agree.
|
||||
resume = inprog[-1]
|
||||
else:
|
||||
resume = next((c for c in cards if c.get("status") != "watched"), None)
|
||||
return {
|
||||
|
||||
+380
-60
@@ -18,26 +18,26 @@ segments. The remux is CPU-light (video copy + a tiny audio transcode), so it ru
|
||||
CPU-only prod host; only P3's full transcode is CPU-heavy.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session as DbSession
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads.edit import encode_thread_cap
|
||||
from app.models import PlexItem
|
||||
from app.plex import paths
|
||||
from app.plex.sync import HW_DECODABLE_CODECS
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
_HLS_ROOT = (
|
||||
Path(settings.plex_hls_dir)
|
||||
if settings.plex_hls_dir
|
||||
@@ -55,9 +55,15 @@ _SEG_SECONDS = 6
|
||||
# worse than the old zero-buffer behaviour).
|
||||
_WARMUP_SEGMENTS = 3
|
||||
_WARMUP_SEG_TIMEOUT_S = 8.0
|
||||
# The HLS variant ffmpeg gives the VIDEO rendition of a multi-audio session. `-var_stream_map` names
|
||||
# each audio rendition (`name:a0`, `a1`, …) but leaves the video one unnamed, so ffmpeg falls back to
|
||||
# its index for that one. See `segment_name` for why this is a constant and not a comment.
|
||||
_VIDEO_VARIANT = "0"
|
||||
_SESSION_IDLE_S = 600 # reap a session with no access for this long
|
||||
# A session directory is named `{rating_key}_{int(start_s)}_{tag}_{aoff:+.2f}` (see start_session);
|
||||
# before multi-audio it was just `{rating_key}_{int(start_s)}`. `sweep_orphans` deletes ONLY names
|
||||
# A session directory is named `{rating_key}_u{user_id}_{int(start_s)}_{tag}_{aoff:+.2f}` (see
|
||||
# start_session); before per-viewer sessions it was `{rating_key}_{int(start_s)}_{tag}_{aoff:+.2f}`,
|
||||
# and before multi-audio just `{rating_key}_{int(start_s)}` — the older shapes stay sweepable, since a
|
||||
# restart leaves them behind like any other. `sweep_orphans` deletes ONLY names
|
||||
# matching one of those two shapes: `_HLS_ROOT` is admin-configurable (PLEX_HLS_DIR), so it may
|
||||
# legitimately point at a shared scratch path we do not own, and a blanket "delete every directory
|
||||
# under the root" would take the neighbours with it.
|
||||
@@ -68,7 +74,9 @@ _SESSION_IDLE_S = 600 # reap a session with no access for this long
|
||||
# non-numeric rating_key would merely leave its directory unswept, which is the safe direction to
|
||||
# fail. Covered by test_plex_stream.py — this regex is the only thing standing between the sweep
|
||||
# and someone else's data.
|
||||
_SESSION_DIR_RE = re.compile(r"^\d+_\d+(_[^_]+_[+-]\d+\.\d{2})?$")
|
||||
_SESSION_DIR_RE = re.compile(
|
||||
r"^\d+_\d+(_[^_]+_[+-]\d+\.\d{2})?$|^\d+_u\d+_\d+_[^_]+_[+-]\d+\.\d{2}$"
|
||||
)
|
||||
# When this process started. `sweep_orphans` only deletes directories older than this: everything
|
||||
# newer belongs to a session THIS run created, and since `start_session` reuses a directory path
|
||||
# for the same (rating_key, offset, tag), "stale name" is not the same as "stale directory".
|
||||
@@ -84,7 +92,16 @@ _PROCESS_START = time.time()
|
||||
_MTIME_GRACE_S = 2.0
|
||||
|
||||
_lock = threading.Lock()
|
||||
_sessions: dict[str, "HlsSession"] = {}
|
||||
# Keyed by VIEWER and item, (user_id, rating_key). Keyed by the item alone, a second account opening
|
||||
# the same film `_kill`ed the first one's session: the displaced viewer got a 404, reclaimed at its
|
||||
# playhead, and so evicted the other in turn — two people could not watch one film at once. (One
|
||||
# account on two devices still shares a session; that case ends in the player's bounded reclaim.)
|
||||
_sessions: dict[tuple[int, str], "HlsSession"] = {}
|
||||
# How many live sessions one account may hold at once (two devices, or a next item starting while the
|
||||
# last one winds down). A new one past it replaces that account's least recently used — whose waiting
|
||||
# session request then returns at once, since `wait_for` watches the killed ffmpeg — so one account
|
||||
# can neither pile up film-sized scratch copies nor park more than this many request workers.
|
||||
_MAX_SESSIONS_PER_USER = 2
|
||||
|
||||
|
||||
class HlsSession:
|
||||
@@ -94,12 +111,21 @@ class HlsSession:
|
||||
directory: Path,
|
||||
proc: subprocess.Popen,
|
||||
start_s: float,
|
||||
entry: str,
|
||||
variants: "Sequence[str]",
|
||||
transcode: bool = False,
|
||||
hw: bool = False,
|
||||
tonemap: bool = False,
|
||||
user_id: int = 0,
|
||||
):
|
||||
self.key = key
|
||||
# Unguessable per-session id, carried in the URLs the client is handed. A seek-restart
|
||||
# replaces the viewer's registered session for the item — and the old player's next request
|
||||
# for `seg_12.ts`, resolved by (viewer, rating_key) alone, was answered 200 with the NEW session's twelfth segment, produced from a different offset
|
||||
# under `-copyts`. Wrong bytes, not an error, so the client's reclaim never fired on it
|
||||
# (deep-review round 5). A request naming a session that is no longer registered is a 404,
|
||||
# which the player recovers from by restarting at the playhead.
|
||||
self.id = secrets.token_urlsafe(9)
|
||||
self.key = key # the item's rating_key (what the logs name)
|
||||
self.user_id = user_id # whose session it is: the registry key is (user_id, key)
|
||||
self.dir = directory
|
||||
self.proc = proc
|
||||
self.transcode = transcode # video re-encoded (non-h264 source), not stream-copied
|
||||
@@ -113,8 +139,16 @@ class HlsSession:
|
||||
# subtitle cue-shift both key off the true content position — no drift. Provisional = X until
|
||||
# the first segment is probed.
|
||||
self.media_start_s = start_s
|
||||
self.entry = entry # the playlist filename hls.js should load (master.m3u8 with subs, else index.m3u8)
|
||||
# The rendition names this session's ffmpeg was told to write — () for a single-audio
|
||||
# session. Held on the session because it is what `artifact_may_arrive` needs to tell a name
|
||||
# this session CAN produce from one belonging to the other argv branch (see it).
|
||||
self.variants = tuple(variants)
|
||||
# The playlist filename hls.js should load: the master of a multi-audio session, else the
|
||||
# single index. Derived, not passed, so it cannot disagree with the argv.
|
||||
self.entry = _MASTER_PL if self.variants else _INDEX_PL
|
||||
self.last_access = time.time()
|
||||
# `artifact_may_arrive` reports a refusing session directory once, not per segment.
|
||||
self.probe_failure_reported = False
|
||||
|
||||
|
||||
def _seek_opts(start_s: float) -> list[str]:
|
||||
@@ -212,7 +246,15 @@ def _probe_video_kind(src: Path) -> str:
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
# The safe kind is a real answer, but it is the answer for an 8-bit SDR source — a blind
|
||||
# ffprobe sends every HDR file down it, so playback quietly degrades to a software
|
||||
# transcode with nothing anywhere saying why. The path stays out of the line: it is the
|
||||
# title of whatever is playing.
|
||||
log.info(
|
||||
"video-kind probe failed; assuming the HW-safe pipeline",
|
||||
extra=logs.event("plex.stream.probe_kind_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return _VK_HWSAFE
|
||||
pix_fmt = bits = transfer = ""
|
||||
for line in out.stdout.splitlines():
|
||||
@@ -241,7 +283,14 @@ def _opencl_present() -> bool:
|
||||
software libx264. Its own function so it's monkeypatchable in tests, mirroring _render_node_present."""
|
||||
try:
|
||||
return any(_OPENCL_ICD_DIR.glob("*.icd"))
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# A MISSING directory does not come through here — `glob` simply yields nothing for one.
|
||||
# This is a directory we cannot read, and the answer it forces (no OpenCL) costs every HDR
|
||||
# source the hardware tonemap path.
|
||||
log.info(
|
||||
"the OpenCL ICD directory could not be read; treating OpenCL as absent",
|
||||
extra=logs.event("plex.stream.opencl_probe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -271,6 +320,30 @@ def _hwaccel_in_opts(hw: bool, tonemap: bool = False) -> list[str]:
|
||||
return ["-hwaccel", "vaapi", "-hwaccel_output_format", "vaapi", "-vaapi_device", _VAAPI_DEVICE]
|
||||
|
||||
|
||||
# Audio is ALWAYS re-encoded on the HLS path: the source is typically AC3/DTS 5.1, which no browser
|
||||
# plays, so it is downmixed to stereo AAC. `-ac 2` uses ffmpeg's standard downmix matrix; nothing is
|
||||
# normalised, which is why a film mastered for a cinema arrives with whispered dialogue and deafening
|
||||
# action (bug hunt 2026-09, item 14).
|
||||
_AUDIO_ENC = ["-c:a", "aac", "-ac", "2", "-b:a", "192k"]
|
||||
# Optional loudness normalisation (admin knob `plex_audio_loudnorm`, default OFF). EBU R128 single-pass
|
||||
# `loudnorm` — two-pass is impossible here, since a seek-restart session must start emitting segments
|
||||
# immediately and cannot analyse the file first.
|
||||
#
|
||||
# The downmix is FORCED FIRST (`aformat=channel_layouts=stereo`) rather than left to `-ac 2`: measuring
|
||||
# 5.1 and then folding it down would normalise something other than what the viewer hears. `loudnorm`
|
||||
# runs its own resampler at 192 kHz internally, so the chain resamples back to 48 kHz for the AAC
|
||||
# encoder. Measured on a real 5.1 AC3 film (Westworld S1E6, 60 s from 15:00): I -27.6 → -16.6 LUFS,
|
||||
# LRA 13.3 → 9.8 LU — the quiet/loud gap narrows, and everything gets ~11 dB louder, so expect to turn
|
||||
# the volume down after switching this on. The filter's 3 s lookahead does NOT desync A/V: measured
|
||||
# under this pipeline's own `-copyts`, the audio start PTS is identical with and without it.
|
||||
_LOUDNORM_AF = "aformat=channel_layouts=stereo,loudnorm=I=-16:TP=-1.5:LRA=11,aresample=48000"
|
||||
|
||||
|
||||
def _aenc_opts(loudnorm: bool) -> list[str]:
|
||||
"""Audio encoder args for one output, with the normalisation filter when the admin enabled it."""
|
||||
return [*_AUDIO_ENC, *(["-af", _LOUDNORM_AF] if loudnorm else [])]
|
||||
|
||||
|
||||
def _venc_opts(
|
||||
transcode: bool,
|
||||
hw: bool,
|
||||
@@ -360,7 +433,7 @@ def _select_encode(db: DbSession, item: PlexItem, transcode: bool, src: Path) ->
|
||||
source that has an OpenCL runtime, while a 10-bit SDR source drops to software."""
|
||||
if not transcode:
|
||||
return (False, False)
|
||||
mode = (sysconfig.get_str(db, "plex_transcode_hw") or "auto").lower()
|
||||
mode = sysconfig.get_str(db, "plex_transcode_hw")
|
||||
if mode == "off" or not _render_node_present():
|
||||
return (False, False)
|
||||
if mode == "on":
|
||||
@@ -403,6 +476,8 @@ def _sw_child_preexec() -> None:
|
||||
playback."""
|
||||
try:
|
||||
os.nice(_SW_TRANSCODE_NICE)
|
||||
# silent-ok: this runs in the forked child, between fork and exec. Logging there can deadlock
|
||||
# on a lock some other thread held at fork time, and the child is about to become ffmpeg.
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
@@ -429,7 +504,14 @@ def _probe_audio_langs(src: Path) -> list[str]:
|
||||
timeout=10,
|
||||
)
|
||||
return [ln.strip() for ln in out.stdout.splitlines()]
|
||||
except (subprocess.SubprocessError, OSError):
|
||||
except (subprocess.SubprocessError, OSError) as exc:
|
||||
# An empty list IS the single-audio path, so a failed probe silently collapses a file with
|
||||
# several audio tracks down to one — and the viewer gets a player with no language picker
|
||||
# rather than an error.
|
||||
log.warning(
|
||||
"audio-track probe failed; falling back to a single-audio session",
|
||||
extra=logs.event("plex.stream.probe_audio_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return []
|
||||
|
||||
|
||||
@@ -446,6 +528,7 @@ def _ffmpeg_cmd(
|
||||
preset: str = "veryfast",
|
||||
crf: int = 20,
|
||||
max_height: int = 0,
|
||||
loudnorm: bool = False,
|
||||
) -> list[str]:
|
||||
# For a transcode session the video is re-encoded (VAAPI or libx264) instead of stream-copied; the
|
||||
# hwaccel decode options go before -i, the encoder args replace `-c:v copy` below. Everything else is
|
||||
@@ -469,11 +552,15 @@ def _ffmpeg_cmd(
|
||||
if audio_langs:
|
||||
# MULTI-RENDITION: map EVERY audio track as an alternate HLS rendition in one group, so hls.js
|
||||
# switches audio CLIENT-SIDE on the same timeline (no session restart, no drift) — the premium
|
||||
# audio-switch path. Produces master.m3u8 + stream_%v.m3u8 (v0=video, v1..=audio) + seg_%v_%d.ts.
|
||||
# audio-switch path. Produces master.m3u8 + one stream_%v.m3u8 and seg_%v_*.ts per variant,
|
||||
# where %v is the video rendition's index ("0") and each audio rendition's `name:` (a0, a1,
|
||||
# …). Those names are spelled by `segment_name` / `variant_playlist_name` below, which the
|
||||
# guard and the warm-up read back — see them for the measurement behind the ordering.
|
||||
args += ["-map", "0:v:0"]
|
||||
for i in range(len(audio_langs)):
|
||||
args += ["-map", f"{ai}:a:{i}?"]
|
||||
args += ["-sn", *venc, "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
|
||||
# One `-af` covers EVERY audio output of this file, so each rendition is normalised.
|
||||
args += ["-sn", *venc, *_aenc_opts(loudnorm)]
|
||||
var = ["v:0,agroup:aud"]
|
||||
for i, lang in enumerate(audio_langs):
|
||||
entry = f"a:{i},agroup:aud,name:a{i}"
|
||||
@@ -482,23 +569,25 @@ def _ffmpeg_cmd(
|
||||
if i == 0:
|
||||
entry += ",default:yes"
|
||||
var.append(entry)
|
||||
# The name patterns come from the vocabulary functions, with ffmpeg's own placeholders passed
|
||||
# through them — so the producer and the parser cannot spell the layout differently (round 7).
|
||||
args += _HLS_TAIL + [
|
||||
"-master_pl_name",
|
||||
"master.m3u8",
|
||||
_MASTER_PL,
|
||||
"-var_stream_map",
|
||||
" ".join(var),
|
||||
"-hls_segment_filename",
|
||||
str(out_dir / "seg_%v_%d.ts"),
|
||||
str(out_dir / "stream_%v.m3u8"),
|
||||
str(out_dir / segment_name("%d", "%v")),
|
||||
str(out_dir / variant_playlist_name("%v")),
|
||||
]
|
||||
else:
|
||||
ao = audio_ord if audio_ord is not None else 0
|
||||
args += ["-map", "0:v:0", "-map", f"{ai}:a:{ao}?", "-sn"]
|
||||
args += [*venc, "-c:a", "aac", "-ac", "2", "-b:a", "192k"]
|
||||
args += [*venc, *_aenc_opts(loudnorm)]
|
||||
args += _HLS_TAIL + [
|
||||
"-hls_segment_filename",
|
||||
str(out_dir / "seg_%d.ts"),
|
||||
str(out_dir / "index.m3u8"),
|
||||
str(out_dir / segment_name("%d")),
|
||||
str(out_dir / _INDEX_PL),
|
||||
]
|
||||
return args
|
||||
|
||||
@@ -531,8 +620,14 @@ def _probe_first_pts(seg: Path) -> float | None:
|
||||
tok = line.strip().strip(",")
|
||||
if tok:
|
||||
return float(tok)
|
||||
except (subprocess.SubprocessError, ValueError, OSError):
|
||||
pass
|
||||
except (subprocess.SubprocessError, ValueError, OSError) as exc:
|
||||
# None means "keep the offset the client asked for", and the whole point of this probe is
|
||||
# that the two differ: without it the subtitle track is shifted by whatever the container
|
||||
# started its timestamps at. Seconds of drift, and nothing on screen calls it a failure.
|
||||
log.warning(
|
||||
"first-PTS probe failed; the requested offset is used as-is",
|
||||
extra=logs.event("plex.stream.first_pts_probe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -541,29 +636,62 @@ def _kill(s: HlsSession) -> None:
|
||||
s.proc.terminate()
|
||||
try:
|
||||
s.proc.wait(timeout=3)
|
||||
# silent-ok: the escalation IS the handling. A transcode that ignores TERM for three
|
||||
# seconds gets KILL, which is exactly what this branch does.
|
||||
except subprocess.TimeoutExpired:
|
||||
s.proc.kill()
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001 — teardown finishes whatever the process does
|
||||
# Not nothing: what survives this is an ffmpeg nobody owns any more, holding a VAAPI
|
||||
# handle and a core, and the transcode cap that is supposed to bound them counts rows in
|
||||
# `_sessions` rather than processes. One line is the only way that ever surfaces.
|
||||
log.info(
|
||||
"could not stop the transcode process",
|
||||
extra=logs.event("plex.stream.kill_failed", key=s.key, reason=type(exc).__name__),
|
||||
)
|
||||
shutil.rmtree(s.dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _enforce_cap(cap: int) -> None:
|
||||
# Caller holds _lock. Only TRANSCODE sessions count toward plex_max_transcodes — remux
|
||||
# (stream-copy) and direct-serve are cheap and deliberately uncapped (see config.py:
|
||||
# plex_max_transcodes, and the user-facing config hint). Drop the least-recently-accessed
|
||||
# transcode sessions over the cap, leaving any concurrent remux streams untouched.
|
||||
tx = [(key, s) for key, s in _sessions.items() if s.transcode]
|
||||
if len(tx) <= cap:
|
||||
return
|
||||
for key, s in sorted(tx, key=lambda kv: kv[1].last_access)[: len(tx) - cap]:
|
||||
_kill(s)
|
||||
_sessions.pop(key, None)
|
||||
def _enforce_caps(user_id: int, max_transcodes: int, max_sessions: int) -> None:
|
||||
"""Caller holds _lock. Three bounds, applied after a session is registered, each dropping the
|
||||
least recently accessed sessions over it:
|
||||
|
||||
* this account's own sessions (`_MAX_SESSIONS_PER_USER`) — first, so a single account filling the
|
||||
box evicts itself before anyone else;
|
||||
* TRANSCODE sessions (plex_max_transcodes): the CPU/GPU-heavy ones;
|
||||
* EVERY session (plex_max_sessions). A remux is cheap on the CPU but not on the disk: each one is a
|
||||
whole segmented copy of the film under `_HLS_ROOT`, written faster than realtime. Counted only
|
||||
by the transcode cap, N distinct items were N ffmpegs and N film-sized scratch directories.
|
||||
|
||||
Somebody's playback stops at each drop. The route turns the missing session into a retryable 404
|
||||
and the player restarts at the playhead, so from the server nothing failed — and "my film keeps
|
||||
rebuffering" has no other explanation anywhere, hence the line."""
|
||||
|
||||
def cull(pool: list[tuple[tuple[int, str], HlsSession]], cap: int, bound: str) -> None:
|
||||
if len(pool) <= cap:
|
||||
return
|
||||
for reg, sess in sorted(pool, key=lambda kv: kv[1].last_access)[: len(pool) - cap]:
|
||||
log.info(
|
||||
"a session was dropped to stay under a cap",
|
||||
extra=logs.event(
|
||||
"plex.stream.session_culled",
|
||||
key=sess.key,
|
||||
bound=bound,
|
||||
cap=cap,
|
||||
sessions=len(pool),
|
||||
),
|
||||
)
|
||||
_kill(sess)
|
||||
_sessions.pop(reg, None)
|
||||
|
||||
cull([kv for kv in _sessions.items() if kv[0][0] == user_id], _MAX_SESSIONS_PER_USER, "user")
|
||||
cull([kv for kv in _sessions.items() if kv[1].transcode], max_transcodes, "transcode")
|
||||
cull(list(_sessions.items()), max_sessions, "total")
|
||||
|
||||
|
||||
def start_session(
|
||||
db: DbSession,
|
||||
item: PlexItem,
|
||||
user_id: int,
|
||||
start_s: float,
|
||||
audio_ord: int | None = None,
|
||||
aoff: float = 0.0,
|
||||
@@ -579,16 +707,27 @@ def start_session(
|
||||
src = paths.local_media_path(db, item.file_path)
|
||||
if src is None:
|
||||
return None
|
||||
# Read once, here: the knob has to be identical for the directory name and the ffmpeg argv, and a
|
||||
# session respawn (the HW->SW fallback) must not pick up a mid-playback config change.
|
||||
loudnorm = sysconfig.get_bool(db, "plex_audio_loudnorm")
|
||||
key = item.rating_key
|
||||
reg = (user_id, key) # the registry key: one session per viewer per item
|
||||
start_s = max(0.0, float(start_s))
|
||||
audio_langs = _probe_audio_langs(src) if multi else []
|
||||
is_multi = len(audio_langs) > 1 # only worth a master playlist when there really are ≥2 tracks
|
||||
tag = "multi" if is_multi else str(audio_ord)
|
||||
directory = _HLS_ROOT / f"{key}_{int(start_s)}_{tag}_{aoff:+.2f}"
|
||||
# The rendition names the argv below will produce — one source for the argv, the warm-up's
|
||||
# segment names, the entry playlist and the artifact guard.
|
||||
variants = session_variants(audio_langs if is_multi else None)
|
||||
# Loudness normalisation changes the AUDIO THE SEGMENTS CONTAIN, so it belongs in the directory
|
||||
# name beside the other parameters that do (offset, audio track, A/V offset): a session left over
|
||||
# from before the admin flipped the knob must not be mistaken for one that honours it. Appended to
|
||||
# the tag with a DASH — `_SESSION_DIR_RE` treats the tag as one underscore-free field, and that
|
||||
# regex is the only thing keeping the orphan sweep off a shared scratch path (see its comment).
|
||||
tag = ("multi" if is_multi else str(audio_ord)) + ("-ln" if loudnorm else "")
|
||||
directory = _HLS_ROOT / f"{key}_u{user_id}_{int(start_s)}_{tag}_{aoff:+.2f}"
|
||||
|
||||
def _seg(i: int) -> Path:
|
||||
# multi-audio: the video rendition is variant v0 → seg_{i}_0.ts; single-audio → seg_{i}.ts
|
||||
return directory / (f"seg_{i}_0.ts" if is_multi else f"seg_{i}.ts")
|
||||
return directory / segment_name(i, _VIDEO_VARIANT if variants else None)
|
||||
|
||||
def _spawn(
|
||||
hw: bool, tonemap: bool = False, *, replace_only: "HlsSession | None" = None
|
||||
@@ -603,7 +742,7 @@ def start_session(
|
||||
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
||||
preset, crf, max_height = "veryfast", 20, 0
|
||||
if transcode and not hw:
|
||||
preset = sysconfig.get_str(db, "plex_transcode_preset") or "veryfast"
|
||||
preset = sysconfig.get_str(db, "plex_transcode_preset")
|
||||
crf = sysconfig.get_int(db, "plex_transcode_crf")
|
||||
max_height = sysconfig.get_int(db, "plex_transcode_max_height")
|
||||
# Nice the SOFTWARE transcode child (not the HW path, and not remux) so a CPU-bound re-encode
|
||||
@@ -611,9 +750,9 @@ def start_session(
|
||||
# platform without fork (the API always runs in a Linux container in prod) it's simply omitted.
|
||||
preexec = _sw_child_preexec if (transcode and not hw and os.name == "posix") else None
|
||||
with _lock:
|
||||
if replace_only is not None and _sessions.get(key) is not replace_only:
|
||||
if replace_only is not None and _sessions.get(reg) is not replace_only:
|
||||
return None # superseded/culled while we waited out of lock — don't clobber the newer one
|
||||
old = _sessions.pop(key, None)
|
||||
old = _sessions.pop(reg, None)
|
||||
if old is not None:
|
||||
_kill(old)
|
||||
shutil.rmtree(directory, ignore_errors=True)
|
||||
@@ -632,6 +771,7 @@ def start_session(
|
||||
preset=preset,
|
||||
crf=crf,
|
||||
max_height=max_height,
|
||||
loudnorm=loudnorm,
|
||||
),
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
@@ -642,15 +782,35 @@ def start_session(
|
||||
directory,
|
||||
proc,
|
||||
start_s,
|
||||
"master.m3u8" if is_multi else "index.m3u8",
|
||||
variants,
|
||||
transcode=transcode,
|
||||
hw=hw,
|
||||
tonemap=tonemap,
|
||||
user_id=user_id,
|
||||
)
|
||||
_sessions[reg] = s
|
||||
# The one line that says which pipeline a session actually got. Everything downstream
|
||||
# — the cap, the HW→SW fallback, the subtitle offset — is about this choice, and until
|
||||
# now it was only inferable from the ffmpeg command nobody keeps.
|
||||
log.info(
|
||||
"an HLS session started",
|
||||
extra=logs.event(
|
||||
"plex.stream.session_started",
|
||||
key=key,
|
||||
transcode=transcode,
|
||||
hw=hw,
|
||||
tonemap=tonemap,
|
||||
variants=len(variants or ()),
|
||||
start_s=round(start_s, 1),
|
||||
),
|
||||
)
|
||||
# Admin-configurable caps (Configuration → Plex); at least 1 each so playback can't be
|
||||
# capped to zero.
|
||||
_enforce_caps(
|
||||
user_id,
|
||||
max(1, sysconfig.get_int(db, "plex_max_transcodes")),
|
||||
max(1, sysconfig.get_int(db, "plex_max_sessions")),
|
||||
)
|
||||
_sessions[key] = s
|
||||
# Admin-configurable concurrency cap (Configuration → Plex → Max concurrent transcodes);
|
||||
# at least 1 so playback can't be capped to zero.
|
||||
_enforce_cap(max(1, sysconfig.get_int(db, "plex_max_transcodes")))
|
||||
return s
|
||||
|
||||
s = _spawn(*_select_encode(db, item, transcode, src))
|
||||
@@ -666,7 +826,8 @@ def start_session(
|
||||
# ONE lock in _spawn, so we can never tear down the newer session. s.hw is now False, so the route
|
||||
# honestly reports "CPU" and the player's Preparing-stream state stays correct.
|
||||
log.warning(
|
||||
"plex transcode: HW session produced no segments — falling back to software key=%s", key
|
||||
"the hardware transcode produced no segments; falling back to software",
|
||||
extra=logs.event("plex.stream.hw_fallback", key=key),
|
||||
)
|
||||
respawned = _spawn(False, False, replace_only=s)
|
||||
if respawned is None:
|
||||
@@ -675,7 +836,7 @@ def start_session(
|
||||
# turns a None return into a "Media file not found on disk" 404 — dishonest for a file that
|
||||
# exists. A (dead) session instead degrades to a retryable 404 ("No active playback session" /
|
||||
# "Not available yet") the client re-requests.
|
||||
return current_session(key) or s
|
||||
return current_session(user_id, key) or s
|
||||
s = respawned
|
||||
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
||||
if ready:
|
||||
@@ -706,14 +867,125 @@ def start_session(
|
||||
return s
|
||||
|
||||
|
||||
def current_session(key: str) -> HlsSession | None:
|
||||
def current_session(user_id: int, key: str) -> HlsSession | None:
|
||||
with _lock:
|
||||
s = _sessions.get(key)
|
||||
s = _sessions.get((user_id, key))
|
||||
if s is not None:
|
||||
s.last_access = time.time()
|
||||
return s
|
||||
|
||||
|
||||
# The names ffmpeg writes into a session directory, in ONE place — and "one place" means the argv
|
||||
# BUILDS its patterns from these same functions (`segment_name("%d", "%v")`), not that a comment
|
||||
# says so. Getting this wrong is invisible in both directions: a name we failed to anticipate used
|
||||
# to fall back to "wait and see" (round 6's HIGH), and now that the rule is positive it falls back
|
||||
# to a 404 instead — so the producer and the parser have to be literally the same spelling.
|
||||
#
|
||||
# MEASURED (ffmpeg 7.1.5 in the API container — the binary that actually writes these files — and
|
||||
# re-checked on 8.1.1): `-hls_segment_filename seg_%v_%d.ts` substitutes `%v` with the VARIANT and
|
||||
# `%d` with the SEQUENCE — variant FIRST. A dual-audio session writes seg_0_0.ts… (video, the
|
||||
# unnamed variant) plus seg_a0_0.ts… / seg_a1_0.ts… (the `name:aN` audio renditions), and the
|
||||
# playlists beside them are master.m3u8 + stream_0/a0/a1.m3u8. Single-audio uses `seg_%d.ts` +
|
||||
# index.m3u8, with no variant field at all.
|
||||
#
|
||||
# The two argv branches are DISJOINT vocabularies, which is why the guard below is keyed to the
|
||||
# SESSION and not to HLS in general: a single-audio session can never write `master.m3u8`,
|
||||
# `stream_*.m3u8` or any `seg_<variant>_<n>.ts`, and a multi-audio one can never write `index.m3u8`
|
||||
# or `seg_<n>.ts`. Accepting the union meant a hopeless name of the other shape was waited out for
|
||||
# the full playlist/segment timeout — the threadpool parking this guard exists to end (round 7).
|
||||
_MASTER_PL = "master.m3u8"
|
||||
_INDEX_PL = "index.m3u8"
|
||||
_SEG_NAME_RE = re.compile(r"^seg_(?:([0-9a-z]{1,4})_)?(\d{1,9})\.ts$")
|
||||
|
||||
|
||||
def session_variants(audio_langs: "Sequence[str | None] | None") -> tuple[str, ...]:
|
||||
"""The `-var_stream_map` rendition names a session will write, or () for a single-audio one.
|
||||
|
||||
The video rendition is left unnamed in the map, so ffmpeg falls back to its index
|
||||
(`_VIDEO_VARIANT`); each audio rendition is named `aN` in track order."""
|
||||
if not audio_langs:
|
||||
return ()
|
||||
return (_VIDEO_VARIANT, *(f"a{i}" for i in range(len(audio_langs))))
|
||||
|
||||
|
||||
def segment_name(seq: "int | str", variant: "str | None" = None) -> str:
|
||||
"""The file ffmpeg writes for the `seq`-th segment of `variant`.
|
||||
|
||||
`variant=None` is a single-audio session, whose argv carries no `%v` field at all. `seq` and
|
||||
`variant` are typed loosely because `_ffmpeg_cmd` builds its own `-hls_segment_filename` pattern
|
||||
through this function, passing ffmpeg's `"%d"` / `"%v"` placeholders — that is what keeps the
|
||||
producer and the parser from drifting, rather than a comment asking the next editor not to."""
|
||||
return f"seg_{variant}_{seq}.ts" if variant else f"seg_{seq}.ts"
|
||||
|
||||
|
||||
def variant_playlist_name(variant: str) -> str:
|
||||
"""The sub-playlist ffmpeg writes for one rendition of a multi-audio session."""
|
||||
return f"stream_{variant}.m3u8"
|
||||
|
||||
|
||||
def playlist_names(variants: "Sequence[str]") -> frozenset[str]:
|
||||
"""Every playlist name this session's argv can write — the master plus one per rendition, or
|
||||
just `index.m3u8` for a single-audio session."""
|
||||
if not variants:
|
||||
return frozenset({_INDEX_PL})
|
||||
return frozenset({_MASTER_PL, *(variant_playlist_name(v) for v in variants)})
|
||||
|
||||
|
||||
def artifact_may_arrive(s: "HlsSession", filename: str) -> bool:
|
||||
"""Whether waiting for `filename` can possibly end in a file OF THIS SESSION.
|
||||
|
||||
Positive rule, not a blocklist: a session produces segments and playlists and NOTHING else, so
|
||||
any other name is refused at once. ffmpeg also produces segments strictly in order, so a segment
|
||||
whose PREDECESSOR does not exist yet is not the next one — it is beyond the generated region,
|
||||
and no amount of waiting will conjure it. Playlists are the one thing that must still be waited
|
||||
for: they are rewritten continuously and have no ordering to read.
|
||||
|
||||
Keyed to the SESSION's own vocabulary, not to HLS's: the two argv branches are disjoint, so a
|
||||
name belonging to the other shape is as hopeless here as a random string. Reading those as "an
|
||||
HLS name, so wait and see" left the whole class reachable — measured through the real route
|
||||
against a live single-audio session, `master.m3u8` and `stream_q7z.m3u8` were 404s after 30.0 s
|
||||
and `seg_a0_0.ts` / `seg_zzzz_0.ts` after 20.1 s each (deep-review round 7).
|
||||
|
||||
Saying no at once matters because the route that waits is a SYNC one: it holds a thread off the
|
||||
~40-slot pool, and a pooled DB connection with it, for the whole timeout. An ordinary seek past
|
||||
the produced region was enough to park one for 20 s, and forty concurrent requests for a name
|
||||
that will never exist parked the whole API (deep-review round 5). Round 5's guard read only
|
||||
single-audio segment names, so both halves of that were still fully reachable: every multi-audio
|
||||
name fell through, and so did any `.m3u8`/`.vtt` name at all."""
|
||||
if filename in playlist_names(s.variants):
|
||||
return True
|
||||
m = _SEG_NAME_RE.match(filename)
|
||||
if m is None:
|
||||
return False
|
||||
variant, seq = m.group(1), int(m.group(2))
|
||||
# The variant field must be one THIS session writes: present and known for a multi-audio
|
||||
# session, absent for a single-audio one.
|
||||
if variant not in (s.variants or (None,)):
|
||||
return False
|
||||
if seq == 0:
|
||||
return True
|
||||
try:
|
||||
return (s.dir / segment_name(seq - 1, variant)).exists()
|
||||
except OSError as exc:
|
||||
# `exists()` answers False for a missing file without raising, so this is the session
|
||||
# directory itself refusing — a full or unmounted disk. Every segment then 404s, which the
|
||||
# player shows as a stall. Once per session: this runs for every segment request and
|
||||
# every hls.js retry of one, and the sink's limiter still stores 30 a minute of a
|
||||
# template — stdout would get them all.
|
||||
if not s.probe_failure_reported:
|
||||
s.probe_failure_reported = True
|
||||
log.warning(
|
||||
"a session segment could not be checked",
|
||||
extra=logs.event(
|
||||
"plex.stream.segment_probe_failed",
|
||||
key=s.key,
|
||||
seq=seq,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def wait_for(path: Path, timeout: float = 20.0, proc: subprocess.Popen | None = None) -> bool:
|
||||
"""Wait until a session file (playlist / segment) exists and is non-empty. Segments are
|
||||
produced ~faster than realtime, so a segment just ahead of playback appears quickly; a segment
|
||||
@@ -727,12 +999,16 @@ def wait_for(path: Path, timeout: float = 20.0, proc: subprocess.Popen | None =
|
||||
try:
|
||||
if path.exists() and path.stat().st_size > 0:
|
||||
return True
|
||||
# silent-ok: this is a 150 ms poll — a line here is a line every 150 ms for the whole
|
||||
# timeout. The final check below decides, and the 404 the caller then returns is honest.
|
||||
except OSError:
|
||||
pass
|
||||
if proc is not None and proc.poll() is not None:
|
||||
# the producer exited — no more segments are coming; one last check then give up.
|
||||
try:
|
||||
return path.exists() and path.stat().st_size > 0
|
||||
# silent-ok: one last look after the producer exited. False here becomes a 404 the
|
||||
# player recovers from, which is the honest answer either way.
|
||||
except OSError:
|
||||
return False
|
||||
time.sleep(0.15)
|
||||
@@ -765,7 +1041,16 @@ def sweep_orphans(older_than: float | None = None) -> int:
|
||||
dropped = 0
|
||||
try:
|
||||
entries = list(_HLS_ROOT.iterdir())
|
||||
except OSError: # the root doesn't exist yet — nothing to sweep
|
||||
# silent-ok: nothing has been transcoded yet, so there is nothing to sweep.
|
||||
except FileNotFoundError:
|
||||
return 0
|
||||
except OSError as exc:
|
||||
# Any OTHER reason the root will not list is not "nothing to sweep": the sweep reports 0
|
||||
# like a clean run while every crashed session's segments stay on disk forever.
|
||||
log.warning(
|
||||
"the HLS root could not be listed; no orphans were swept",
|
||||
extra=logs.event("plex.stream.sweep_root_unreadable", reason=type(exc).__name__),
|
||||
)
|
||||
return 0
|
||||
for p in entries:
|
||||
if not _SESSION_DIR_RE.match(p.name):
|
||||
@@ -775,7 +1060,9 @@ def sweep_orphans(older_than: float | None = None) -> int:
|
||||
try:
|
||||
if p.stat().st_mtime >= cutoff:
|
||||
continue # this run made (or reused) it — never ours to delete
|
||||
except OSError: # vanished under us (a concurrent restart of that session)
|
||||
# silent-ok: vanished under us — a concurrent restart of that session removed it, which
|
||||
# is the outcome this sweep wanted anyway.
|
||||
except OSError:
|
||||
continue
|
||||
with _lock:
|
||||
if any(s.dir == p for s in _sessions.values()):
|
||||
@@ -785,14 +1072,47 @@ def sweep_orphans(older_than: float | None = None) -> int:
|
||||
return dropped
|
||||
|
||||
|
||||
def reap_idle() -> int:
|
||||
# How long a session whose ffmpeg has FINISHED may go unfetched before a new session request reaps
|
||||
# it (see reap_idle).
|
||||
_FINISHED_IDLE_S = 30
|
||||
|
||||
|
||||
def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int:
|
||||
"""Kill sessions nobody has said they still want for `_SESSION_IDLE_S` — or, once their ffmpeg
|
||||
has finished, for `finished_idle_s`.
|
||||
|
||||
A finished session is also what a PAUSED viewer has: a remux runs far faster than realtime, so
|
||||
ffmpeg is done a minute or two in and writes ENDLIST; hls.js then stops asking for the playlist,
|
||||
and with its forward buffer full a paused player fetches nothing at all. With the 30 s rule that
|
||||
killed a one-minute pause, and playback died ~30 s after resuming (deep-review round 4).
|
||||
|
||||
What makes the short rule safe again is that fetching is no longer the only signal: a player
|
||||
parked MID-ITEM checkpoints its position every 10 s, and that route calls `touch_session`, so a
|
||||
viewer who is merely paused says so far more often than this rule asks (round 5). The heartbeat
|
||||
stops once the item is finished or before a position is known (`usePlexProgress.flush`) — both
|
||||
states where reaping is correct, and where the player's 404 → reclaim recovers a viewer who
|
||||
scrubs back in anyway. Do not restate this as "every 10 s whether or not it is playing": round 6
|
||||
found that claim in two comments and not in the code. Round 4's blanket
|
||||
`reap_abandoned` bought the same safety by keeping every finished session for ten minutes — each
|
||||
one a whole segmented copy of the film on a volume shared with the download center."""
|
||||
now = time.time()
|
||||
dropped = 0
|
||||
with _lock:
|
||||
for key, s in list(_sessions.items()):
|
||||
for reg, s in list(_sessions.items()):
|
||||
done = s.proc.poll() is not None
|
||||
if now - s.last_access > _SESSION_IDLE_S or (done and now - s.last_access > 30):
|
||||
idle = now - s.last_access
|
||||
if idle > _SESSION_IDLE_S or (done and idle > finished_idle_s):
|
||||
_kill(s)
|
||||
_sessions.pop(key, None)
|
||||
_sessions.pop(reg, None)
|
||||
dropped += 1
|
||||
return dropped
|
||||
|
||||
|
||||
def touch_session(user_id: int, key: str) -> None:
|
||||
"""Say this viewer is still there for `key`, without fetching anything. `last_access` used to move
|
||||
only on an HLS file fetch, which a paused player does not do — so the only way to keep a paused
|
||||
viewer's session alive was to keep EVERY abandoned one just as long."""
|
||||
with _lock:
|
||||
s = _sessions.get((user_id, key))
|
||||
if s is not None:
|
||||
s.last_access = time.time()
|
||||
|
||||
+972
-17
File diff suppressed because it is too large
Load Diff
+88
-31
@@ -15,20 +15,20 @@ dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are
|
||||
item-detail/player path (P2), not here.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||
from app.plex import detail, people
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.genres import hu_genres
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
||||
_BROWSER_VIDEO = {"h264"}
|
||||
_BROWSER_AUDIO = {"aac", "mp3", "opus", "vorbis"}
|
||||
@@ -102,6 +102,9 @@ def _video_stream_props(meta: dict) -> tuple[int | None, bool]:
|
||||
bd = s.get("bitDepth")
|
||||
try:
|
||||
depth = int(bd) if bd else None
|
||||
# silent-ok: unknown bit depth, which is also what most listings report. The
|
||||
# playback class then falls back to the codec alone, and per-stream noise over a
|
||||
# whole library would drown the sync's own lines.
|
||||
except (TypeError, ValueError):
|
||||
depth = None
|
||||
hdr = str(s.get("colorTrc") or "").lower() in _HDR_TRANSFERS
|
||||
@@ -112,6 +115,8 @@ def _video_stream_props(meta: dict) -> tuple[int | None, bool]:
|
||||
def _epoch(v) -> datetime | None:
|
||||
try:
|
||||
return datetime.fromtimestamp(int(v), tz=timezone.utc) if v else None
|
||||
# silent-ok: a field Plex leaves out of most listing rows. None is a legitimate stored value,
|
||||
# and every caller treats it as "not known" rather than as an error.
|
||||
except (TypeError, ValueError, OSError):
|
||||
return None
|
||||
|
||||
@@ -120,6 +125,7 @@ def _date(v):
|
||||
"""Parse a Plex 'YYYY-MM-DD' string (originallyAvailableAt) → date, or None."""
|
||||
try:
|
||||
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
|
||||
# silent-ok: as `_epoch` above — an absent or unparsable air date is stored as no air date.
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@@ -135,6 +141,7 @@ def _tags(meta: dict, key: str, limit: int | None = None) -> list | None:
|
||||
def _rating(meta: dict) -> float | None:
|
||||
try:
|
||||
return float(meta.get("audienceRating") or meta.get("rating"))
|
||||
# silent-ok: TypeError on `float(None)` is the ordinary path — most items carry no rating.
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@@ -266,15 +273,19 @@ def sync(db: Session) -> dict:
|
||||
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
||||
try:
|
||||
stats["people_enriched"] = _enrich_people_batch(db, plex)
|
||||
except Exception as e: # best-effort — never fail the sync over the backfill
|
||||
log.warning("Plex people enrich batch failed: %s", e)
|
||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
||||
# With the traceback: a blind catch over a batch that fetches AND writes can be
|
||||
# a Plex shape change, a DB error or our own bug, and `%s` told them apart badly.
|
||||
log.exception("the Plex people-enrich batch failed")
|
||||
# Probe a bounded batch of HW-decodable transcode items for bit-depth/HDR, so the
|
||||
# playback-class filter's HW/SW split is precise (a 10-bit/HDR HEVC → software) over
|
||||
# successive runs. Only HW-decodable-codec transcodes are probed — the rest are SW by codec.
|
||||
try:
|
||||
stats["media_probed"] = _probe_media_batch(db, plex)
|
||||
except Exception as e: # best-effort — never fail the sync over the backfill
|
||||
log.warning("Plex media probe batch failed: %s", e)
|
||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
||||
log.exception("the Plex media-probe batch failed")
|
||||
# silent-ok: the return value IS the report. `{"skipped": …}` goes to the scheduler, which
|
||||
# logs the job result, and to the admin route, which answers it to the caller.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
@@ -285,8 +296,8 @@ def sync(db: Session) -> dict:
|
||||
# appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata.
|
||||
try:
|
||||
stats["people_pruned"] = people.prune_orphan_people(db)
|
||||
except Exception as e: # best-effort housekeeping — never fail the sync over it
|
||||
log.warning("Plex people prune failed: %s", e)
|
||||
except Exception: # noqa: BLE001 — best-effort housekeeping; never fail the sync over it
|
||||
log.exception("the Plex people prune failed")
|
||||
log.info("Plex sync done: %s", stats)
|
||||
return stats
|
||||
|
||||
@@ -325,7 +336,9 @@ def _probe_media_batch(db: Session, plex: PlexClient, limit: int = _PROBE_BATCH)
|
||||
for row in rows:
|
||||
try:
|
||||
meta = plex.metadata(row.rating_key) or {}
|
||||
except Exception as e: # transient — leave media_probed_at NULL so it retries next run
|
||||
# No traceback and only debug: this is per item over a whole library, and the stack is
|
||||
# the same `plex.metadata` call every time. The retry next run is the handling.
|
||||
except Exception as e: # noqa: BLE001 — transient; leave media_probed_at NULL to retry
|
||||
log.debug("Plex media probe fetch failed for %s: %s", row.rating_key, e)
|
||||
continue
|
||||
bit_depth, hdr = _video_stream_props(meta)
|
||||
@@ -372,7 +385,8 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
|
||||
for row in [*movies, *shows]:
|
||||
try:
|
||||
meta = plex.metadata(row.rating_key) or {}
|
||||
except Exception as e:
|
||||
# As in `_probe_media_batch` above: per item, so the message without the stack.
|
||||
except Exception as e: # noqa: BLE001 — transient; leave people_enriched_at NULL
|
||||
log.debug("Plex people enrich fetch failed for %s: %s", row.rating_key, e)
|
||||
continue # leave people_enriched_at NULL so it's retried on a later run
|
||||
people.enrich_row_people(db, row, people.cast_from_meta(meta))
|
||||
@@ -436,6 +450,10 @@ def _apply_facet_fields(row, meta: dict) -> None:
|
||||
row.studio = (meta.get("studio") or None) and str(meta.get("studio"))[:255]
|
||||
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
|
||||
row.genres = _tags(meta, "Genre")
|
||||
# Every row has Hungarian genres, not only one with an HU overlay twin (the overlay pass, which runs
|
||||
# after this, puts the twin's own list over it): the HU genre facets and filter read this column,
|
||||
# and an English word there could never match a filter read in Hungarian.
|
||||
row.genres_hu = hu_genres(row.genres)
|
||||
row.directors = _tags(meta, "Director")
|
||||
# The listing carries only the top ~20 cast, so re-applying it would DOWNGRADE an already-enriched
|
||||
# full cast (up to 24, grown by the rich backfill/info-page enrich), silently breaking the
|
||||
@@ -647,35 +665,72 @@ def _sync_overlay(
|
||||
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
||||
) -> None:
|
||||
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/genres differ
|
||||
by language; the language-neutral people_text feeds both search vectors.
|
||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/tagline/genres
|
||||
differ by language; the language-neutral people_text feeds both search vectors. A show section's
|
||||
episodes are overlaid too (E30): the HU section lists every episode with its Hungarian title and
|
||||
summary, and the episode rows live in the EN show library.
|
||||
|
||||
This is a full RECONCILE: every guid-bearing EN row's HU fields are cleared first, then the HU
|
||||
section repopulates the matches — so an EN row whose HU twin vanished (or was re-guid'd) falls back
|
||||
to EN instead of keeping stale Hungarian metadata. HU items with no EN match are skipped."""
|
||||
if lib.kind == "movie":
|
||||
rows = db.execute(
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.guid.is_not(None),
|
||||
passes = [
|
||||
(
|
||||
_TYPE_MOVIE,
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.guid.is_not(None),
|
||||
),
|
||||
)
|
||||
).scalars()
|
||||
item_type = _TYPE_MOVIE
|
||||
]
|
||||
else:
|
||||
rows = db.execute(
|
||||
select(PlexShow).where(PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None))
|
||||
).scalars()
|
||||
item_type = _TYPE_SHOW
|
||||
passes = [
|
||||
(
|
||||
_TYPE_SHOW,
|
||||
select(PlexShow).where(
|
||||
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
|
||||
),
|
||||
),
|
||||
(
|
||||
_TYPE_EPISODE,
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "episode",
|
||||
PlexItem.guid.is_not(None),
|
||||
),
|
||||
),
|
||||
]
|
||||
for item_type, rows_q in passes:
|
||||
_overlay_pass(db, plex, lib, item_type, db.execute(rows_q).scalars(), stats)
|
||||
db.flush()
|
||||
|
||||
|
||||
def _overlay_pass(
|
||||
db: Session,
|
||||
plex: PlexClient,
|
||||
lib: PlexLibrary,
|
||||
item_type: int,
|
||||
rows: Iterable[PlexItem] | Iterable[PlexShow],
|
||||
stats: dict,
|
||||
) -> None:
|
||||
"""One kind of the overlay (movies, shows or episodes) — see `_sync_overlay`."""
|
||||
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
||||
by_guid: dict[str, list] = {}
|
||||
by_guid: dict[str, list[PlexItem | PlexShow]] = {}
|
||||
for r in rows:
|
||||
by_guid.setdefault(r.guid, []).append(r)
|
||||
if r.guid: # the passes select guid-bearing rows only
|
||||
by_guid.setdefault(r.guid, []).append(r)
|
||||
# Reconcile step: clear HU fields on all guid-bearing rows. Only rows that actually held HU data
|
||||
# become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is cheap; matches are re-set below.
|
||||
# The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking one, from
|
||||
# its own English list — otherwise the HU genre list mixed "Drama" (no twin) with "Dráma".
|
||||
episodes = item_type == _TYPE_EPISODE
|
||||
for group in by_guid.values():
|
||||
for row in group:
|
||||
row.title_hu = row.summary_hu = row.genres_hu = None
|
||||
row.title_hu = row.summary_hu = None
|
||||
if not episodes:
|
||||
row.tagline_hu = None
|
||||
row.genres_hu = hu_genres(row.genres)
|
||||
seen: set[str] = (
|
||||
set()
|
||||
) # first HU entry for a guid wins; a dup HU item can't overwrite or double-count
|
||||
@@ -687,13 +742,15 @@ def _sync_overlay(
|
||||
seen.add(g)
|
||||
title_hu = (meta.get("title") or "").strip() or None
|
||||
summary_hu = meta.get("summary")
|
||||
genres_hu = _tags(meta, "Genre")
|
||||
tagline_hu = (meta.get("tagline") or "").strip() or None
|
||||
genres_hu = hu_genres(_tags(meta, "Genre"))
|
||||
for row in targets:
|
||||
row.title_hu = title_hu
|
||||
row.summary_hu = summary_hu
|
||||
row.genres_hu = genres_hu
|
||||
if not episodes:
|
||||
row.tagline_hu = tagline_hu
|
||||
row.genres_hu = genres_hu or row.genres_hu
|
||||
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
||||
db.flush()
|
||||
|
||||
|
||||
def _upsert_collection(
|
||||
|
||||
+199
-32
@@ -10,21 +10,23 @@ One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `
|
||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import cast
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import PlexItem, PlexPerson, PlexShow
|
||||
from app.plex.cards import display_title
|
||||
from app.plex.people import people_by_credits, warm_portraits
|
||||
|
||||
log = logging.getLogger("siftlode.tmdb")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||
_MIN_INTERVAL = 0.03
|
||||
|
||||
@@ -126,9 +128,17 @@ class TmdbClient:
|
||||
"""Resolve an external id (imdb_id / tvdb_id) to TMDB items via /find."""
|
||||
return self._get(f"/find/{external_id}", {"external_source": source})
|
||||
|
||||
def countries(self) -> list[dict] | None:
|
||||
"""TMDB's country list: `[{iso_3166_1, english_name, native_name}]` (a JSON array, which
|
||||
`_get`'s dict annotation does not describe)."""
|
||||
return cast("list[dict] | None", self._get("/configuration/countries"))
|
||||
|
||||
def fetch_person(self, person_id: int) -> dict | None:
|
||||
"""One person's bio + combined credits (E5 S6), in a single append_to_response request."""
|
||||
return self._get(f"/person/{person_id}", {"append_to_response": "combined_credits"})
|
||||
"""One person's bio + combined credits (E5 S6) + the bio's translations (E30 S4), in a single
|
||||
append_to_response request."""
|
||||
return self._get(
|
||||
f"/person/{person_id}", {"append_to_response": "combined_credits,translations"}
|
||||
)
|
||||
|
||||
|
||||
def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int | None:
|
||||
@@ -142,6 +152,12 @@ def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int
|
||||
try:
|
||||
return int(ext)
|
||||
except ValueError:
|
||||
# The caller stamps the watermark on a None, so this title is never looked at again:
|
||||
# one unreadable guid costs it its artwork and its cast permanently.
|
||||
log.info(
|
||||
"a Plex guid named TMDB but did not carry a numeric id",
|
||||
extra=logs.event("plex.tmdb.guid_unparsable", provider=provider),
|
||||
)
|
||||
return None
|
||||
source = {"imdb": "imdb_id", "tvdb": "tvdb_id"}.get(provider)
|
||||
if not source:
|
||||
@@ -250,8 +266,12 @@ def parse_tmdb(payload: dict, media_type: str) -> dict:
|
||||
c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")
|
||||
][:8]
|
||||
or None,
|
||||
# The ISO code beside TMDB's English name (E30): the client names the country in the reader's
|
||||
# language from the code; the name is the fallback for a code it cannot name.
|
||||
"production_countries": [
|
||||
c.get("name") for c in (payload.get("production_countries") or []) if c.get("name")
|
||||
{"code": c.get("iso_3166_1") or None, "name": c["name"]}
|
||||
for c in (payload.get("production_countries") or [])
|
||||
if c.get("name")
|
||||
]
|
||||
or None,
|
||||
"recommendations": recs or None,
|
||||
@@ -297,8 +317,8 @@ def _link_people(db: Session, links: dict[str, int]) -> None:
|
||||
|
||||
|
||||
def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the remainder —
|
||||
mirrors _enrich_people_batch). A transient TmdbError leaves the watermark NULL for retry; a
|
||||
"""Enrich up to `limit` not-yet-TMDB-enriched titles (movies first, shows fill the
|
||||
remainder). A transient TmdbError leaves the watermark NULL for retry; a
|
||||
definitive no-match (unresolvable guid / TMDB 404) still stamps the watermark so it isn't re-fetched
|
||||
every run. One commit at the end. Returns the number processed (matched OR definitively unmatched)."""
|
||||
movies = (
|
||||
@@ -352,13 +372,62 @@ def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||
return processed
|
||||
|
||||
|
||||
def _country(c: dict | str) -> dict:
|
||||
"""One stored production country as served: `{code, name}`. A row enriched before E30 stored the
|
||||
bare English name; until `backfill_country_codes` reaches it, it serves with no code (the client
|
||||
shows the name)."""
|
||||
return c if isinstance(c, dict) else {"code": None, "name": c}
|
||||
|
||||
|
||||
def backfill_country_codes(db: Session, tmdb: TmdbClient) -> int:
|
||||
"""Give the countries stored before E30 (bare English names) their ISO codes, from TMDB's own
|
||||
country list — the names came from that list, so they match it exactly. One request, and only
|
||||
while such a row is left: re-fetching each title instead would take the enrichment batch days.
|
||||
A name the list does not hold keeps no code. Returns the number of rows rewritten."""
|
||||
stale = [
|
||||
row
|
||||
for model in (PlexItem, PlexShow)
|
||||
for row in db.execute(
|
||||
select(model).where(func.jsonb_typeof(model.production_countries[0]) == "string")
|
||||
).scalars()
|
||||
]
|
||||
if not stale:
|
||||
return 0
|
||||
codes = {
|
||||
c["english_name"]: c["iso_3166_1"]
|
||||
for c in tmdb.countries() or []
|
||||
if c.get("english_name") and c.get("iso_3166_1")
|
||||
}
|
||||
for row in stale:
|
||||
row.production_countries = [
|
||||
{"code": codes.get(c), "name": c} if isinstance(c, str) else c
|
||||
for c in row.production_countries or []
|
||||
]
|
||||
db.commit()
|
||||
return len(stale)
|
||||
|
||||
|
||||
def run_enrich(db: Session) -> dict:
|
||||
"""Scheduler entry point. No-op (never an error) when the TMDB key isn't configured."""
|
||||
if not sysconfig.get_str(db, "tmdb_api_key"):
|
||||
return {"enriched": 0, "skipped": "tmdb_api_key not set"}
|
||||
with TmdbClient(db) as tmdb:
|
||||
coded = backfill_country_codes(db, tmdb)
|
||||
n = enrich_batch(db, tmdb, settings.tmdb_enrich_batch)
|
||||
return {"enriched": n}
|
||||
return {"enriched": n, **({"countries_coded": coded} if coded else {})}
|
||||
|
||||
|
||||
def run_people_enrich(db: Session) -> dict:
|
||||
"""Scheduler entry point of people_enrich (E30 S6): a batch of person bios (the most-credited
|
||||
first) and a batch of portraits. Each half is a no-op — never an error — without its TMDB key /
|
||||
without Plex."""
|
||||
batch = settings.people_enrich_batch
|
||||
if sysconfig.get_str(db, "tmdb_api_key"):
|
||||
with TmdbClient(db) as tmdb:
|
||||
bios = enrich_people_batch(db, tmdb, batch)
|
||||
else:
|
||||
bios = {"skipped": "tmdb_api_key not set"}
|
||||
return {"bios": bios, "portraits": warm_portraits(db, batch)}
|
||||
|
||||
|
||||
def resolve_library_guids(
|
||||
@@ -402,10 +471,11 @@ def resolve_library_guids(
|
||||
return movies, shows
|
||||
|
||||
|
||||
def resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]:
|
||||
def resolve_recs(db: Session, recs: list[dict] | None, *, lang: str) -> list[dict]:
|
||||
"""Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:<id>`, bucketed
|
||||
by media_type — see resolve_library_guids) and, if so, its rating_key — so the client can deep-link
|
||||
the ones we have and merely display the rest."""
|
||||
the ones we have and merely display the rest. One we have is named as the library names it, in
|
||||
`lang` (E30); one we lack keeps the title TMDB gave."""
|
||||
if not recs:
|
||||
return []
|
||||
movies, shows = resolve_library_guids(db, recs)
|
||||
@@ -413,13 +483,19 @@ def resolve_recs(db: Session, recs: list[dict] | None) -> list[dict]:
|
||||
for r in recs:
|
||||
g = f"tmdb:{r.get('tmdb_id')}"
|
||||
row = shows.get(g) if r.get("media_type") == "tv" else movies.get(g)
|
||||
named = {"title": display_title(row, lang)} if row is not None else {}
|
||||
out.append(
|
||||
{**r, "in_library": row is not None, "rating_key": row.rating_key if row else None}
|
||||
{
|
||||
**r,
|
||||
**named,
|
||||
"in_library": row is not None,
|
||||
"rating_key": row.rating_key if row else None,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def serve_block(db: Session, row) -> dict | None:
|
||||
def serve_block(db: Session, row, *, lang: str) -> dict | None:
|
||||
"""The TMDB extras for a movie/show detail response, served from the row (no live TMDB call).
|
||||
None until the row has been TMDB-enriched (the client then shows nothing extra)."""
|
||||
if row.tmdb_enriched_at is None:
|
||||
@@ -433,30 +509,58 @@ def serve_block(db: Session, row) -> dict | None:
|
||||
"keywords": row.keywords or [],
|
||||
"crew": row.crew or [],
|
||||
"studios": row.studios or [],
|
||||
"production_countries": row.production_countries or [],
|
||||
"recommendations": resolve_recs(db, row.recommendations),
|
||||
"production_countries": [_country(c) for c in row.production_countries or []],
|
||||
"recommendations": resolve_recs(db, row.recommendations, lang=lang),
|
||||
"backdrops": row.backdrops or [],
|
||||
}
|
||||
|
||||
|
||||
# --- Person enrichment (E5 S6) -------------------------------------------------------------------
|
||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once (the first person-page open)
|
||||
# and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once — by the people_enrich job, or
|
||||
# by the first person-page open if the job has not reached them yet — and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||
# endpoint resolves them to library rows by guid at serve time (like recommendations).
|
||||
|
||||
_KNOWN_FOR_MAX = 15
|
||||
# The candidate pool kept per person. Only the titles that are in the library are shown (at most
|
||||
# routes.plex.people._KNOWN_FOR_SHOWN), so the pool is deeper than the rail: a star's 15 most-voted
|
||||
# credits are often not the 15 a given library holds.
|
||||
_KNOWN_FOR_MAX = 40
|
||||
# TMDB genre ids of the shows a person visits as a guest (News, Reality, Talk). Ranked by popularity,
|
||||
# talk-show visits crowded out the films (Tom Cruise: 14 of 15 were talk shows; E30 S6).
|
||||
_GUEST_GENRES = {10763, 10764, 10767}
|
||||
# A credit in which the person plays themselves ("Self", "Himself - Guest", "Herself (archive footage)").
|
||||
_SELF_CREDIT = re.compile(r"^\s*(self|himself|herself|themselves)\b", re.IGNORECASE)
|
||||
# The crew jobs that make a title the person's own work (a director is known for what they directed).
|
||||
_KNOWN_FOR_CREW_JOBS = {"Director", "Creator"}
|
||||
|
||||
|
||||
def _known_for_credit(c: dict, crew: bool) -> bool:
|
||||
if crew:
|
||||
return c.get("job") in _KNOWN_FOR_CREW_JOBS
|
||||
if _SELF_CREDIT.match(c.get("character") or ""):
|
||||
return False
|
||||
return not _GUEST_GENRES.intersection(c.get("genre_ids") or [])
|
||||
|
||||
|
||||
def parse_person(payload: dict) -> dict:
|
||||
"""TMDB /person/{id} (+combined_credits) → the persisted bio fields. `known_for` = the person's
|
||||
most-popular movie/tv credits (deduped by id), top `_KNOWN_FOR_MAX`, resolved to library later."""
|
||||
credits = (payload.get("combined_credits") or {}).get("cast") or []
|
||||
most-voted movie/tv credits — roles and directed works, not guest visits as themselves — deduped,
|
||||
the top `_KNOWN_FOR_MAX`, resolved to library rows at serve time. Ranked by vote count, as TMDB's
|
||||
own "Known for" is: popularity is this week's attention, which a talk-show visit wins."""
|
||||
combined = payload.get("combined_credits") or {}
|
||||
credits = [c for c in combined.get("cast") or [] if _known_for_credit(c, crew=False)] + [
|
||||
c for c in combined.get("crew") or [] if _known_for_credit(c, crew=True)
|
||||
]
|
||||
# Dedup on (id, media_type), not the bare id: TMDB movie ids and tv ids are independent sequences
|
||||
# (id 603 can be both a movie and a show — see resolve_library_guids), so a same-id movie+show pair
|
||||
# is two distinct credits and both must survive.
|
||||
seen: set[tuple[int, str]] = set()
|
||||
known: list[dict] = []
|
||||
for c in sorted(credits, key=lambda c: c.get("popularity") or 0, reverse=True):
|
||||
ranked = sorted(
|
||||
credits,
|
||||
key=lambda c: (c.get("vote_count") or 0, c.get("popularity") or 0),
|
||||
reverse=True,
|
||||
)
|
||||
for c in ranked:
|
||||
tid, mt = c.get("id"), c.get("media_type")
|
||||
if not tid or mt not in ("movie", "tv") or (tid, mt) in seen:
|
||||
continue
|
||||
@@ -466,6 +570,7 @@ def parse_person(payload: dict) -> dict:
|
||||
break
|
||||
return {
|
||||
"biography": (payload.get("biography") or "").strip() or None,
|
||||
"biography_hu": _translated_bio(payload, "hu"),
|
||||
"birthday": payload.get("birthday") or None,
|
||||
"deathday": payload.get("deathday") or None,
|
||||
"place_of_birth": payload.get("place_of_birth") or None,
|
||||
@@ -475,8 +580,17 @@ def parse_person(payload: dict) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _translated_bio(payload: dict, lang: str) -> str | None:
|
||||
"""The `lang` biography from the appended `translations` (TMDB leaves an untranslated one blank)."""
|
||||
for tr in (payload.get("translations") or {}).get("translations") or []:
|
||||
if tr.get("iso_639_1") == lang:
|
||||
return ((tr.get("data") or {}).get("biography") or "").strip() or None
|
||||
return None
|
||||
|
||||
|
||||
def apply_person(row: PlexPerson, parsed: dict) -> None:
|
||||
row.biography = parsed["biography"]
|
||||
row.biography_hu = parsed["biography_hu"]
|
||||
row.birthday = parsed["birthday"]
|
||||
row.deathday = parsed["deathday"]
|
||||
row.place_of_birth = parsed["place_of_birth"]
|
||||
@@ -486,11 +600,22 @@ def apply_person(row: PlexPerson, parsed: dict) -> None:
|
||||
row.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _fetch_person_into(tmdb: TmdbClient, person: PlexPerson) -> None:
|
||||
"""Fetch one person from TMDB and write the row (not committed). A 404 (deleted person) stamps the
|
||||
watermark with empty fields so it isn't fetched again. Raises TmdbError on a transient failure,
|
||||
leaving the row untouched and `tmdb_enriched_at` NULL for a retry."""
|
||||
payload = tmdb.fetch_person(cast(int, person.tmdb_person_id))
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
else:
|
||||
apply_person(person, parse_person(payload))
|
||||
|
||||
|
||||
def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
"""Lazily fetch + persist one person's TMDB bio (first person-page open), committing the write so it
|
||||
survives regardless of the read path. No-op (False) without a key or a `tmdb_person_id`; a transient
|
||||
error leaves `tmdb_enriched_at` NULL so the next open retries. A 404 (deleted person) stamps the
|
||||
watermark with empty fields so we don't refetch it every open."""
|
||||
"""Lazily fetch + persist one person's TMDB bio (a person page opened before the background job
|
||||
reached them), committing the write so it survives regardless of the read path. No-op (False)
|
||||
without a key or a `tmdb_person_id`; a transient error leaves `tmdb_enriched_at` NULL so the next
|
||||
open retries."""
|
||||
if not person.tmdb_person_id or not sysconfig.get_str(db, "tmdb_api_key"):
|
||||
return False
|
||||
try:
|
||||
@@ -498,12 +623,54 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
# longer blocks a serve; still cap the outbound TMDB call with a short timeout so a hanging TMDB
|
||||
# doesn't tie up a threadpool worker for the default 30s.
|
||||
with TmdbClient(db, timeout=8.0) as tmdb:
|
||||
payload = tmdb.fetch_person(person.tmdb_person_id)
|
||||
except TmdbError:
|
||||
_fetch_person_into(tmdb, person)
|
||||
except TmdbError as exc:
|
||||
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
||||
# front of the user right now shows an empty biography with nothing to say why.
|
||||
log.warning(
|
||||
"the TMDB person lookup failed",
|
||||
extra=logs.event(
|
||||
"plex.tmdb.person_enrich_failed",
|
||||
person_id=person.id,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
return False
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
else:
|
||||
apply_person(person, parse_person(payload))
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
# Lookups failing this many people in a row end a people batch: TMDB itself is degraded, and hammering
|
||||
# it helps nobody. One person failing alone is skipped, so an id TMDB keeps refusing never stalls the rest.
|
||||
_PEOPLE_FAILS_IN_A_ROW = 3
|
||||
|
||||
|
||||
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
||||
"""Fetch up to `limit` linked, not-yet-enriched people (E30 S6), the most-credited first, so a
|
||||
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
||||
lands: a page open during the run already sees it. A failed lookup is skipped (left unstamped for the
|
||||
next run); _PEOPLE_FAILS_IN_A_ROW of them in a row end the batch."""
|
||||
ids = people_by_credits(db, "bio", limit)
|
||||
done = failed = in_a_row = 0
|
||||
for pid in ids:
|
||||
if in_a_row >= _PEOPLE_FAILS_IN_A_ROW:
|
||||
break
|
||||
person = db.get(PlexPerson, pid)
|
||||
# A page open may have enriched this person since the select: skip, don't fetch twice.
|
||||
if person is None or person.tmdb_enriched_at is not None:
|
||||
continue
|
||||
try:
|
||||
_fetch_person_into(tmdb, person)
|
||||
except TmdbError as exc:
|
||||
db.rollback()
|
||||
failed, in_a_row = failed + 1, in_a_row + 1
|
||||
log.warning(
|
||||
"a TMDB person lookup in the batch failed",
|
||||
extra=logs.event(
|
||||
"plex.tmdb.people_batch_failed", person_id=pid, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
continue
|
||||
db.commit()
|
||||
done, in_a_row = done + 1, 0
|
||||
return {"enriched": done, **({"failed": failed} if failed else {})}
|
||||
|
||||
@@ -17,14 +17,17 @@ and the incremental pull resolves a genuine divergence to Plex's state (see `_pu
|
||||
running a last-write-wins comparison.
|
||||
"""
|
||||
|
||||
import logging
|
||||
# On the PlexError lines in this module: none of them carries a traceback, deliberately. A
|
||||
# PlexError is httpx's own message — the method, the URL and the status of one call — and the stack
|
||||
# beneath it is always this `except` plus `PlexClient._get`. The attrs beside the line say which
|
||||
# user and which phase, which is the part the message cannot know.
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.db import SessionLocal
|
||||
from app.models import PlexItem, PlexLink, PlexState
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
@@ -36,8 +39,7 @@ from app.plex.sync import (
|
||||
_paginate,
|
||||
)
|
||||
|
||||
log = logging.getLogger("siftlode.plex.watch")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# A resume position below this many seconds isn't worth mirroring — matches the player's own
|
||||
# _PROGRESS_MIN_S (a stray few-second seek is not "in progress").
|
||||
_PROGRESS_MIN_S = 5
|
||||
@@ -148,6 +150,8 @@ def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
||||
link.initial_import_done = True
|
||||
link.last_watch_sync_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
# silent-ok: the `{"skipped": …}` return is the report — the admin route answers it and the
|
||||
# scheduler logs it on the job line.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
@@ -326,8 +330,18 @@ def _resolve_account_id(db: Session, plex: PlexClient, link: PlexLink) -> int:
|
||||
if int(a.get("id") or -1) == 1:
|
||||
username = a.get("name")
|
||||
break
|
||||
except PlexError:
|
||||
pass
|
||||
except PlexError as exc:
|
||||
# The fallback account id 1 is right for the admin-token setup, so sync keeps working —
|
||||
# but the username is never stored, and the UI then has no name to show for the link,
|
||||
# exactly as if the server had not offered one.
|
||||
log.info(
|
||||
"the Plex account list could not be read; the link keeps the default account",
|
||||
extra=logs.event(
|
||||
"plex.watch.accounts_probe_failed",
|
||||
user_id=link.user_id,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
link.plex_account_id = acct_id
|
||||
if username:
|
||||
link.plex_username = username
|
||||
@@ -566,6 +580,7 @@ def run_plex_watch_sync(db: Session) -> dict:
|
||||
total["repushed"] += _repush_dirty(db, plex, link)
|
||||
link.last_watch_sync_at = now
|
||||
db.commit()
|
||||
# silent-ok: the `{"skipped": …}` return is the report, as above.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
@@ -617,6 +632,7 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
||||
total["repushed"] += _repush_dirty(db, plex, link)
|
||||
link.last_watch_sync_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
# silent-ok: the `{"skipped": …}` return is the report, as above.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
|
||||
@@ -37,6 +37,7 @@ class QuotaAction:
|
||||
PLAYLISTS_DELETE = "playlists_delete"
|
||||
VIDEOS_BACKFILL_RECENT = "videos_backfill_recent"
|
||||
VIDEOS_BACKFILL_FULL = "videos_backfill_full"
|
||||
VIDEOS_FEED_FALLBACK = "videos_feed_fallback"
|
||||
VIDEOS_ENRICH = "videos_enrich"
|
||||
VIDEOS_LOOKUP = "videos_lookup"
|
||||
VIDEOS_SEARCH = "videos_search"
|
||||
|
||||
+67
-11
@@ -1,21 +1,30 @@
|
||||
"""A tiny in-process sliding-window rate limiter.
|
||||
|
||||
Generic groundwork: keyed by an arbitrary string (e.g. a client IP), so it can throttle any
|
||||
Generic groundwork: keyed by an arbitrary string (e.g. a client — `client_key`), so it can throttle any
|
||||
endpoint. Single-worker uvicorn → in-memory state is sufficient; it resets on restart, which
|
||||
is fine for abuse throttling (not for anything that must survive a deploy). Not shared across
|
||||
processes — if we ever run multiple workers, swap the backing store for Redis behind the same
|
||||
``allow()`` interface.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import threading
|
||||
import time
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
def __init__(self, max_events: int, window_seconds: float):
|
||||
"""At most `max_events` attempts per `window_seconds` per key — or, when each attempt passes a
|
||||
`cost`, at most that much total cost (a byte budget counts bytes, not calls)."""
|
||||
|
||||
def __init__(self, max_events: float, window_seconds: float):
|
||||
self.max_events = max_events
|
||||
self.window = window_seconds
|
||||
self._hits: dict[str, list[float]] = {}
|
||||
# key → (monotonic time, cost) of every counted attempt still inside the window
|
||||
self._hits: dict[str, list[tuple[float, float]]] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def reset(self) -> None:
|
||||
@@ -25,27 +34,74 @@ class RateLimiter:
|
||||
with self._lock:
|
||||
self._hits.clear()
|
||||
|
||||
def allow(self, key: str) -> bool:
|
||||
"""Record an attempt for ``key`` and return whether it is within the limit.
|
||||
def exhausted(self, key: str) -> bool:
|
||||
"""Whether ``key``'s window is already full (``allow`` would refuse it), WITHOUT counting an
|
||||
attempt — for a limit that counts only some outcomes: ask it first, ``allow`` the ones that
|
||||
count."""
|
||||
cutoff = time.monotonic() - self.window
|
||||
with self._lock:
|
||||
return sum(c for t, c in self._hits.get(key, []) if t > cutoff) >= self.max_events
|
||||
|
||||
True -> under the cap (the attempt is counted).
|
||||
False -> the cap for the current window is already reached (attempt NOT counted, so a
|
||||
def allow(self, key: str, cost: float = 1) -> bool:
|
||||
"""Record an attempt costing ``cost`` for ``key`` and return whether it is within the limit.
|
||||
|
||||
True -> within the cap (the attempt is counted).
|
||||
False -> the attempt would pass the cap for the current window (attempt NOT counted, so a
|
||||
blocked caller can't keep pushing the window forward)."""
|
||||
now = time.monotonic()
|
||||
cutoff = now - self.window
|
||||
with self._lock:
|
||||
hits = [t for t in self._hits.get(key, []) if t > cutoff]
|
||||
if len(hits) >= self.max_events:
|
||||
hits = [(t, c) for t, c in self._hits.get(key, []) if t > cutoff]
|
||||
if sum(c for _, c in hits) + cost > self.max_events:
|
||||
self._hits[key] = hits
|
||||
return False
|
||||
hits.append(now)
|
||||
hits.append((now, cost))
|
||||
self._hits[key] = hits
|
||||
# Opportunistic cleanup so abandoned keys don't accumulate unboundedly.
|
||||
if len(self._hits) > 4096:
|
||||
for k in list(self._hits):
|
||||
fresh = [t for t in self._hits[k] if t > cutoff]
|
||||
fresh = [(t, c) for t, c in self._hits[k] if t > cutoff]
|
||||
if fresh:
|
||||
self._hits[k] = fresh
|
||||
else:
|
||||
del self._hits[k]
|
||||
return True
|
||||
|
||||
|
||||
def _norm_ip(s: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
||||
"""Parse an IP for comparison, unwrapping IPv4-mapped IPv6 (`::ffff:10.0.0.1` → `10.0.0.1`) so a
|
||||
plain-IPv4 allowlist still matches a peer that surfaces in mapped form. None for a non-IP string."""
|
||||
try:
|
||||
ip = ipaddress.ip_address(s.strip())
|
||||
# silent-ok: parsing an untrusted header value, where "not an IP" is a documented answer this
|
||||
# function returns — and a line per request is what an attacker would aim for.
|
||||
except ValueError:
|
||||
return None
|
||||
return ip.ipv4_mapped if getattr(ip, "ipv4_mapped", None) else ip
|
||||
|
||||
|
||||
def client_key(request: Request) -> str:
|
||||
"""Best-effort client identity for rate limiting: its IP, or for IPv6 the /64 the address sits in — a
|
||||
subscriber is handed a whole /64, so keyed on the full address one client would have endless identities
|
||||
to spread its attempts over. X-Forwarded-For is client-controlled, so we trust it
|
||||
ONLY when the request actually arrived from a configured reverse proxy (settings.trusted_proxy_ips
|
||||
— the address the proxy connects FROM, e.g. the VPS Caddy's WireGuard peer IP). Our proxy APPENDS
|
||||
the client it saw to XFF, so the RIGHTMOST entry is the real client and can't be forged by a
|
||||
client pre-seeding the header. Any other peer (someone hitting the app port directly, bypassing the
|
||||
proxy) is untrusted — we use its real socket IP, so it can't spoof its rate-limit identity.
|
||||
|
||||
SAFETY: this relies on `request.client` being the TRUE socket peer. uvicorn is started WITHOUT
|
||||
`--proxy-headers` (see entrypoint.sh) precisely so it never rewrites `request.client` from XFF —
|
||||
do NOT add that flag, or the trust check below would compare against a forgeable value."""
|
||||
peer = request.client.host if request.client else "unknown"
|
||||
client = peer
|
||||
trusted = {_norm_ip(p) for p in settings.trusted_proxy_ip_set} - {None}
|
||||
if _norm_ip(peer) in trusted:
|
||||
xff = request.headers.get("x-forwarded-for")
|
||||
parts = [p.strip() for p in (xff or "").split(",") if p.strip()]
|
||||
if parts:
|
||||
client = parts[-1]
|
||||
ip = _norm_ip(client)
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
return str(ipaddress.IPv6Network((ip, 64), strict=False))
|
||||
return client
|
||||
|
||||
+13
-6
@@ -13,12 +13,12 @@ mutated from a single thread.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from starlette.websockets import WebSocket
|
||||
from app import logs
|
||||
|
||||
log = logging.getLogger("siftlode.realtime")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
@@ -42,9 +42,16 @@ class ConnectionManager:
|
||||
for ws in list(self._conns.get(user_id, ())):
|
||||
try:
|
||||
await ws.send_json(payload)
|
||||
except Exception:
|
||||
# Stale socket; it'll be cleaned up by its own disconnect handler.
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001 — one dead socket must not stop the others
|
||||
# Stale socket; it'll be cleaned up by its own disconnect handler. But this is
|
||||
# also where a live update goes missing: the UI simply never changes, and there
|
||||
# is no request behind it whose failure anyone could look up.
|
||||
log.debug(
|
||||
"a live update could not be pushed to a socket",
|
||||
extra=logs.event(
|
||||
"realtime.push_failed", user_id=user_id, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
|
||||
def push(self, user_id: int, payload: dict) -> None:
|
||||
"""Schedule a push onto the WebSocket event loop. Safe to call from a sync route
|
||||
@@ -54,7 +61,7 @@ class ConnectionManager:
|
||||
return
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(self._push(user_id, payload), loop)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 — a closed loop or socket must not fail the caller
|
||||
log.debug("ws push failed", exc_info=True)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""A request the app refuses, said in a way the UI can put in the reader's language.
|
||||
|
||||
Every refusal the API answers carries a structured `detail` — `{code, reason, …params}` — and the client
|
||||
translates it (`lib/api/core.ts` `localizeDetail`). A bare English string reached the Hungarian UI
|
||||
verbatim: the blocking error dialog, and every inline surface that printed `HttpError.detail`, showed
|
||||
whatever sentence the route happened to hand-write. `refused` is the generic family
|
||||
(`errors.refused.<reason>`, the params interpolated into the sentence); the older specific codes
|
||||
(`not_found`, `youtube`, `playlist`, the admin refusals, …) stay, since each is already translated.
|
||||
|
||||
A test (`test_http_details.py`) reads every `HTTPException` in the app and fails on a string detail,
|
||||
and the frontend's contract test requires every `refused(…, "<reason>")` to be named in both
|
||||
languages — so the rule is enforced, not described.
|
||||
"""
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
def refused(status: int, reason: str, **params: object) -> HTTPException:
|
||||
"""The refusal to raise: `raise refused(409, "view_name_taken", name=name)`. `params` travel in
|
||||
the detail for the sentence to interpolate (`{{name}}`); keep them to values the reader may see."""
|
||||
return HTTPException(status_code=status, detail={"code": "refused", "reason": reason, **params})
|
||||
|
||||
|
||||
# `sysconfig.InvalidSetting.reason` → the refusal it answers as. A dict of literals, so the frontend's
|
||||
# contract test can read every reason this can produce.
|
||||
_SETTING_REASONS = {
|
||||
"not_a_number": "setting_not_a_number",
|
||||
"too_low": "setting_too_low",
|
||||
"too_high": "setting_too_high",
|
||||
"not_a_choice": "setting_not_a_choice",
|
||||
"no_encryption_key": "no_encryption_key",
|
||||
}
|
||||
|
||||
|
||||
def setting_refused(exc) -> HTTPException:
|
||||
"""A value the admin entered that `sysconfig.set_value` refused (`exc`: its `InvalidSetting`)."""
|
||||
return refused(400, _SETTING_REASONS[exc.reason], key=exc.key, **exc.params)
|
||||
+112
-12
@@ -6,23 +6,37 @@ and "fetch a row this user owns or 404" idioms have one definition, one status c
|
||||
one existence-leak policy.
|
||||
"""
|
||||
|
||||
from typing import Any, TypeVar
|
||||
from typing import IO, Any, TypeVar
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi import HTTPException, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import Base
|
||||
from app.models import User
|
||||
from app.issues import gitea
|
||||
from app.refusal import refused
|
||||
from app.youtube.client import YouTubeError, refusal_reason
|
||||
|
||||
M = TypeVar("M", bound=Base)
|
||||
|
||||
|
||||
def get_or_404(db: Session, model: type[M], id_: Any, *, detail: str = "Not found") -> M:
|
||||
def not_found(what: str = "generic") -> HTTPException:
|
||||
"""The 404 for a row that is gone — structured, so the bilingual UI says it in the reader's
|
||||
language (`errors.notFound.<what>`, generic fallback). This is the 404 a user actually meets: the
|
||||
thing was deleted in another tab or on another device. It used to be a bare English string
|
||||
("Unknown view", "Not found"), which the Hungarian UI toasted verbatim."""
|
||||
return HTTPException(status_code=404, detail={"code": "not_found", "reason": what})
|
||||
|
||||
|
||||
def get_or_404(
|
||||
db: Session, model: type[M], id_: Any, *, what: str = "generic", lock: bool = False
|
||||
) -> M:
|
||||
"""Fetch a row by primary key or raise 404. Replaces the `db.get(Model, id); if None: 404`
|
||||
idiom repeated across routes."""
|
||||
obj = db.get(model, id_)
|
||||
idiom repeated across routes. `lock` takes it FOR UPDATE (bypassing the identity map), so two
|
||||
requests that read-then-write its children (a playlist's item positions) run one after the other."""
|
||||
obj = db.get(model, id_, with_for_update=True if lock else None)
|
||||
if obj is None:
|
||||
raise HTTPException(status_code=404, detail=detail)
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
|
||||
@@ -33,13 +47,99 @@ def owned_or_404(
|
||||
user: User,
|
||||
*,
|
||||
owner_attr: str = "user_id",
|
||||
detail: str = "Not found",
|
||||
what: str = "generic",
|
||||
lock: bool = False,
|
||||
) -> M:
|
||||
"""Fetch a row by primary key that `user` owns, or raise 404. A row owned by someone
|
||||
"""Fetch a row by primary key that `user` owns, or raise 404 (`lock`: see `get_or_404`). A row owned by someone
|
||||
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
||||
Generalizes the near-identical per-route ownership helpers (e.g. plex's
|
||||
`_own_playlist_or_404`, the saved-views owner check)."""
|
||||
obj = get_or_404(db, model, id_, detail=detail)
|
||||
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
|
||||
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
|
||||
obj = get_or_404(db, model, id_, what=what, lock=lock)
|
||||
if getattr(obj, owner_attr) != user.id:
|
||||
raise HTTPException(status_code=404, detail=detail)
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
|
||||
def youtube_refusal(exc: Exception) -> HTTPException:
|
||||
"""The answer to an action YouTube refused: a 422 whose reason the UI translates
|
||||
(`errors.youtube.<reason>`). One definition, because the routes had grown three shapes of it —
|
||||
a bare English string quoting the upstream message, a playlist-scoped reason, and nothing at
|
||||
all (a raw 500 from the manual syncs). The upstream failure is already logged where it
|
||||
happened (`youtube.api.error`). A failure that is not the Data API's (the search scraper's) is
|
||||
YouTube being unavailable to us — the retryable answer."""
|
||||
reason = refusal_reason(exc) if isinstance(exc, YouTubeError) else "unavailable"
|
||||
return HTTPException(status_code=422, detail={"code": "youtube", "reason": reason})
|
||||
|
||||
|
||||
async def capped_body(request: Request, limit: int) -> bytes:
|
||||
"""The request body, refused (413 `too_large`) the moment it passes `limit` bytes while still
|
||||
arriving. Counted, not taken from Content-Length: a declared length can be absent (chunked) or
|
||||
wrong, and counting stops reading at the same point a declared-length check would have."""
|
||||
body = bytearray()
|
||||
async for chunk in request.stream():
|
||||
body += chunk
|
||||
if len(body) > limit:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
return bytes(body)
|
||||
|
||||
|
||||
async def capped_stream_to(
|
||||
request: Request, out: IO[bytes], limit: int, refusal: HTTPException
|
||||
) -> int:
|
||||
"""`capped_body`'s sibling for a body too large to hold: the same counting rule, written
|
||||
straight to `out` instead of accumulated, returning the size.
|
||||
|
||||
Two deliberate differences from `capped_body`. It never builds a `bytearray` — the upload route
|
||||
admits 500 MiB by default, so holding the body would trade a disk problem for a memory one. And
|
||||
the refusal is the CALLER'S, passed in rather than assumed: `capped_body`'s flat 413 is right
|
||||
for a screenshot, while the upload route answers 422 `upload_too_large` carrying the admin
|
||||
knob's own number, which is what its dialog shows the user. One counting rule, two answers,
|
||||
neither of them a second copy of the loop.
|
||||
|
||||
The caller owns `out` and owns cleaning it up: a body refused half-way has already written what
|
||||
arrived before the limit was passed.
|
||||
|
||||
**The refusal DRAINS the rest of the body before it is raised, and that is not politeness.**
|
||||
Answering mid-upload and closing the connection is what the client sees as a network failure,
|
||||
not as a refusal: the browser's XHR fires `onerror` before it can read the response, so a file
|
||||
over the limit reported itself as "the server couldn't be reached" instead of naming the limit —
|
||||
measured in a real browser, not reasoned about. Reading the remainder to nowhere lets the client
|
||||
finish sending, read the 422 and say the true thing.
|
||||
|
||||
The bytes are discarded, never written, and the whole request is already bounded by
|
||||
`bodylimit`'s per-route ceiling — so the worst case is bandwidth an authenticated caller was
|
||||
sending anyway, against the old behaviour which wrote exactly those bytes to disk."""
|
||||
size = 0
|
||||
over = False
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if over:
|
||||
continue # draining — see the docstring; counted, not stored
|
||||
if size > limit:
|
||||
over = True
|
||||
continue
|
||||
out.write(chunk)
|
||||
if over:
|
||||
raise refusal
|
||||
return size
|
||||
|
||||
|
||||
def issue_refused(exc: gitea.GiteaError) -> HTTPException:
|
||||
"""The refusal an issue-tracker failure (E16) answers as — the send, the preview and the
|
||||
admin's connection test alike. Literal reasons, one per case: the frontend's contract test reads
|
||||
them to require a sentence for each in both languages."""
|
||||
match exc.reason:
|
||||
case "not_configured":
|
||||
return refused(400, "issues_not_configured")
|
||||
case "unreachable":
|
||||
return refused(422, "issues_unreachable")
|
||||
case "bad_token":
|
||||
return refused(422, "issues_bad_token")
|
||||
case "repo_not_found":
|
||||
return refused(422, "issues_repo_not_found")
|
||||
case "public_repo":
|
||||
return refused(422, "issues_public_repo")
|
||||
case "issues_disabled":
|
||||
return refused(422, "issues_disabled")
|
||||
case _:
|
||||
return refused(422, "issues_rejected")
|
||||
|
||||
+44
-42
@@ -39,6 +39,7 @@ from app.routes.auth import (
|
||||
send_verification_link,
|
||||
)
|
||||
from app.utils import canonical_email, iso, valid_email
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
|
||||
@@ -169,17 +170,17 @@ def _activate_user_by_email(db: Session, email: str) -> None:
|
||||
user = next((u for u in candidates if u.email == email), None) or next(iter(candidates), None)
|
||||
if user is not None and not user.is_active:
|
||||
user.is_active = True
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
|
||||
|
||||
def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
inv = get_or_404(db, Invite, invite_id, detail="No such invite")
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
# Idempotency guard: only a still-pending invite can be decided. Without this a repeat/stale
|
||||
# approve or deny re-runs its side effects — re-activating a user, or (now that deny emails)
|
||||
# sending a duplicate decision mail, even flipping an already-approved invite to "denied" and
|
||||
# mailing a rejection to an already-active account.
|
||||
if inv.status != "pending":
|
||||
raise HTTPException(status_code=409, detail="Invite already decided")
|
||||
raise refused(409, "invite_decided")
|
||||
inv.status = "approved" if approved else "denied"
|
||||
inv.decided_at = datetime.now(timezone.utc)
|
||||
inv.decided_by = admin.email
|
||||
@@ -191,7 +192,7 @@ def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
target_id=inv.id,
|
||||
summary=inv.email,
|
||||
)
|
||||
db.commit()
|
||||
db.flush() # the handler commits, with the activation that goes with it
|
||||
return inv
|
||||
|
||||
|
||||
@@ -210,6 +211,9 @@ def approve_invite(
|
||||
link, has_account, hours = _welcome_mail_parts(db, inv.email)
|
||||
lang = mailtext.lang_for_email(db, inv.email)
|
||||
background.add_task(email_mod.send_access_approved, inv.email, lang, link, has_account, hours)
|
||||
# Before the response, so the mail (a background task, after it) never describes a decision
|
||||
# the database does not hold yet.
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [inv.email])))
|
||||
|
||||
|
||||
@@ -226,6 +230,7 @@ def deny_invite(
|
||||
background.add_task(
|
||||
email_mod.send_access_denied, inv.email, mailtext.lang_for_email(db, inv.email)
|
||||
)
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [inv.email])))
|
||||
|
||||
|
||||
@@ -242,7 +247,7 @@ def add_invite(
|
||||
"""Manually whitelist an email (approved straight away). Convenience for the admin."""
|
||||
email = body.email.strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
# Reuse a canonically-equal row instead of adding a second one for the same mailbox. Prod carries
|
||||
# exactly that pair for one person from 2026-06-16 (hartmann.peter1986@ / hartmannpeter1986@),
|
||||
# which is how an admin ends up approving the same human twice and still seeing them bounce.
|
||||
@@ -259,11 +264,13 @@ def add_invite(
|
||||
inv.status = "approved"
|
||||
inv.decided_at = datetime.now(timezone.utc)
|
||||
inv.decided_by = admin.email
|
||||
db.flush() # a new invite's id, for the audit row — before it, the row named no target
|
||||
audit.record(
|
||||
db, admin.id, AuditAction.INVITE_ADD, target_type="invite", target_id=inv.id, summary=email
|
||||
)
|
||||
db.commit()
|
||||
# One transaction: the whitelisting and the activation it implies land together or not at all.
|
||||
_activate_user_by_email(db, email)
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [email])))
|
||||
|
||||
|
||||
@@ -291,7 +298,7 @@ def send_invitation(
|
||||
|
||||
Only an APPROVED row can be invited: there is nothing to invite someone to while their request is
|
||||
undecided, and inviting a denied address would contradict the decision."""
|
||||
inv = get_or_404(db, Invite, invite_id, detail="No such invite")
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
if inv.status != "approved":
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -364,12 +371,9 @@ def delete_invite(
|
||||
falls back to the env ALLOWED_EMAILS / ADMIN_EMAILS bootstrap lists once no invite exists, so for
|
||||
an address on those the deletion leaves access intact (and an ADMIN_EMAILS address is still
|
||||
promoted to admin on its next Google sign-in). The admin has to hear that, not guess it."""
|
||||
inv = get_or_404(db, Invite, invite_id, detail="No such invite")
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
if _accounts_for(db, [inv.email]):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="That address already has an account — delete the account instead.",
|
||||
)
|
||||
raise refused(409, "address_has_account")
|
||||
still_allowed_by_env = _env_allowed(inv.email)
|
||||
audit.record(
|
||||
db,
|
||||
@@ -408,6 +412,8 @@ def _serialize_user(u: User) -> dict:
|
||||
# confusing sign-in.
|
||||
"google_email": u.google_email if u.google_email != u.email else None,
|
||||
"created_at": iso(u.created_at),
|
||||
# NULL = no sign-in since the column existed (0085), or the shared demo account.
|
||||
"last_login_at": iso(u.last_login_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -433,21 +439,19 @@ def set_user_role(
|
||||
account, and can't remove the last remaining admin. Emails the user when it actually changes."""
|
||||
role = body.role
|
||||
if role not in ("user", "admin"):
|
||||
raise HTTPException(status_code=400, detail="role must be 'user' or 'admin'")
|
||||
target = db.get(User, user_id)
|
||||
if target is None:
|
||||
raise HTTPException(status_code=404, detail="No such user")
|
||||
raise refused(400, "invalid_role")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account's role can't be changed.")
|
||||
raise refused(400, "demo_role")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="You can't change your own role.")
|
||||
raise refused(400, "own_role")
|
||||
if (
|
||||
target.role == "admin"
|
||||
and role == "user"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't remove the last admin.")
|
||||
raise refused(400, "last_admin")
|
||||
changed = target.role != role
|
||||
if changed:
|
||||
audit.record(
|
||||
@@ -491,20 +495,18 @@ def set_user_suspended(
|
||||
un-suspend the user is emailed they can return (suspend is announced reactively, on a blocked
|
||||
sign-in)."""
|
||||
suspended = body.suspended
|
||||
target = db.get(User, user_id)
|
||||
if target is None:
|
||||
raise HTTPException(status_code=404, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be suspended.")
|
||||
raise refused(400, "demo_suspend")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="You can't suspend your own account.")
|
||||
raise refused(400, "own_suspend")
|
||||
if (
|
||||
suspended
|
||||
and target.role == "admin"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't suspend the last active admin.")
|
||||
raise refused(400, "last_active_admin")
|
||||
was_suspended = target.is_suspended
|
||||
if was_suspended != suspended:
|
||||
audit.record(
|
||||
@@ -537,21 +539,17 @@ def admin_delete_user(
|
||||
"""Admin-triggered account deletion — the same full erasure a user can perform on themselves
|
||||
(cascade delete of all personal data + access-request cleanup + Google-grant revoke). Guards:
|
||||
not the demo account, not yourself (use Settings → Account), and not the last admin."""
|
||||
target = db.get(User, user_id)
|
||||
if target is None:
|
||||
raise HTTPException(status_code=404, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be deleted.")
|
||||
raise refused(400, "demo_delete")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Delete your own account from Settings → Account."
|
||||
)
|
||||
raise refused(400, "own_delete")
|
||||
if (
|
||||
target.role == "admin"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't delete the last admin.")
|
||||
raise refused(400, "last_admin")
|
||||
# Record BEFORE the purge — the target row (and its email) is gone afterwards.
|
||||
audit.record(
|
||||
db,
|
||||
@@ -632,7 +630,7 @@ def user_access(
|
||||
Its own endpoint rather than extra columns on /users: `is_allowed` and `can_activate` each run a
|
||||
query, and the panel opens for ONE user at a time — folding them into the list would pay that cost
|
||||
per row on every load to answer a question nobody asked yet."""
|
||||
target = get_or_404(db, User, user_id, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
return _serialize_user(target) | {
|
||||
"is_allowed": is_allowed(db, target.email),
|
||||
"can_activate": can_activate(db, target),
|
||||
@@ -657,7 +655,7 @@ def create_set_password_link(
|
||||
an instance with no mail — narrowing it to one of those would only make the admin's real case the
|
||||
one it refuses. The demo account is the single exception: it is shared, so handing anyone a way to
|
||||
change its password is handing them everyone else's session."""
|
||||
target = get_or_404(db, User, user_id, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "setpw", "reason": "demo", "email": target.email}
|
||||
@@ -693,9 +691,9 @@ def activate_user(
|
||||
This endpoint changes nothing about what an unauthenticated caller can cause — the admin could
|
||||
already do exactly this by re-entering the address in the whitelist box (`add_invite` →
|
||||
`_activate_user_by_email`). What was missing was any way to know they had to."""
|
||||
target = get_or_404(db, User, user_id, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be activated.")
|
||||
raise refused(400, "demo_activate")
|
||||
if target.is_active:
|
||||
return _serialize_user(target) # idempotent: a double click is not an error
|
||||
if not is_allowed(db, target.email):
|
||||
@@ -732,7 +730,7 @@ def resend_user_verification(
|
||||
|
||||
Not rate-limited: an admin is not a threat model to enumerate against, and the deliberate silence
|
||||
of the public endpoint is exactly what this exists to escape."""
|
||||
target = get_or_404(db, User, user_id, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo or target.email_verified:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
@@ -792,7 +790,7 @@ def add_demo_whitelist(
|
||||
sign-in). Idempotent — re-adding an existing email just returns it."""
|
||||
email = body.email.strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
row = db.execute(select(DemoWhitelist).where(DemoWhitelist.email == email)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = DemoWhitelist(
|
||||
@@ -836,10 +834,14 @@ def reset_demo(admin: User = Depends(admin_user), db: Session = Depends(get_db))
|
||||
demo = get_or_create_demo_user(db)
|
||||
seeded = reset_demo_state(db, demo)
|
||||
audit.record(
|
||||
db, admin.id, AuditAction.DEMO_RESET, summary=f"{seeded} sample playlists re-seeded"
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.DEMO_RESET,
|
||||
summary=f"{seeded} sample playlists re-seeded",
|
||||
after={"playlists_seeded": seeded}, # the count as data: named in the reader's language
|
||||
)
|
||||
db.commit()
|
||||
return {"reset": True, "playlists_seeded": seeded}
|
||||
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
||||
|
||||
|
||||
@router.post("/purge-discovery")
|
||||
@@ -858,4 +860,4 @@ def purge_discovery(admin: User = Depends(admin_user), db: Session = Depends(get
|
||||
after=result,
|
||||
)
|
||||
db.commit()
|
||||
return result
|
||||
return {"ok": True, **result}
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""The admin diagnostics console's read side (E22.6): query `log_events`, the server's stored log.
|
||||
|
||||
Admin-only and read-only — rows leave the table through the retention job (`log_retention_days`),
|
||||
never through here. Reading is not audited: it changes nothing, and the console's live tail polls,
|
||||
so a row per read would bury the audit log's real entries.
|
||||
|
||||
What a row holds was redacted when it was written (`logs.record_fields`, and the ingest route for a
|
||||
browser's rows), so this serves the columns as they are; a value the redactor took out reads as
|
||||
`[redacted]` / `[email]`, which is the point — the console shows that something was there.
|
||||
|
||||
Paging is a keyset over `(ts desc, id desc)`: a browser row's `ts` is clamped into its consent
|
||||
window, so `ts` alone is not unique, and an offset would skip or repeat rows while the writer adds
|
||||
new ones at the top. The live tail re-reads the first page rather than asking for "ids after N":
|
||||
several processes write this table, and a batch's ids are taken when it inserts, not when it
|
||||
commits, so a slower batch can land below an id the tail already passed — and would never show.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import Text, and_, cast, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logsink
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import LogEvent, User
|
||||
from app.utils import like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/logs", tags=["admin"])
|
||||
|
||||
PAGE_MAX = 200
|
||||
# A facet list is a chip row, not a report: past this many distinct events the rest are reachable
|
||||
# through the free-text search, and the response stays small whatever a self-host logs.
|
||||
FACET_EVENTS_MAX = 100
|
||||
|
||||
Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
# The filters a facet can leave out of its own count (`LogFilters.conditions`).
|
||||
Facet = Literal["levels", "sources", "user", "event"]
|
||||
# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous
|
||||
# request) — a real value to filter on, not the absence of a filter.
|
||||
NONE = "none"
|
||||
|
||||
|
||||
def _utc(value: datetime | None) -> datetime | None:
|
||||
"""A bound without a zone is UTC — the column is `timestamptz`, and a naive value would be read
|
||||
in the database session's zone instead."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _encode_cursor(row: LogEvent) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps([row.ts.isoformat(), row.id]).encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(cursor: str) -> tuple[datetime, int]:
|
||||
try:
|
||||
ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
return _utc(datetime.fromisoformat(ts)), int(row_id)
|
||||
except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc:
|
||||
raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc
|
||||
|
||||
|
||||
def _accounts(db: Session, user_ids: set[int]) -> dict[str, str]:
|
||||
"""The accounts a page names, by id → address. An id with no account left (erased after the
|
||||
line was written) is simply absent, and the client shows the bare id."""
|
||||
if not user_ids:
|
||||
return {}
|
||||
rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all()
|
||||
return {str(uid): email for uid, email in rows}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LogFilters:
|
||||
"""The console's filters, shared by the page and its facet counts."""
|
||||
|
||||
levels: list[str]
|
||||
sources: list[str]
|
||||
user: str | None
|
||||
event: str | None
|
||||
request_id: str | None
|
||||
job_id: int | None
|
||||
q: str | None
|
||||
since: datetime | None
|
||||
until: datetime | None
|
||||
|
||||
def conditions(self, skip: Facet | None = None) -> list:
|
||||
"""The WHERE conditions, ANDed by the caller. `skip` leaves one filter out: a facet counts
|
||||
its own values under every OTHER filter, so picking a level does not zero the other
|
||||
levels' counts (the way back), while a request pivot does narrow them (honest counts)."""
|
||||
out = []
|
||||
if self.levels and skip != "levels":
|
||||
out.append(LogEvent.level.in_(self.levels))
|
||||
if self.sources and skip != "sources":
|
||||
out.append(LogEvent.source.in_(self.sources))
|
||||
if self.user is not None and skip != "user":
|
||||
out.append(
|
||||
LogEvent.user_id.is_(None)
|
||||
if self.user == NONE
|
||||
else LogEvent.user_id == int(self.user)
|
||||
)
|
||||
if self.event is not None and skip != "event":
|
||||
out.append(
|
||||
LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event
|
||||
)
|
||||
if self.request_id is not None:
|
||||
out.append(LogEvent.request_id == self.request_id)
|
||||
if self.job_id is not None:
|
||||
out.append(LogEvent.job_id == self.job_id)
|
||||
if self.since is not None:
|
||||
out.append(LogEvent.ts >= _utc(self.since))
|
||||
if self.until is not None:
|
||||
out.append(LogEvent.ts < _utc(self.until))
|
||||
if self.q and self.q.strip():
|
||||
like = f"%{like_escape(self.q.strip())}%"
|
||||
out.append(
|
||||
or_(
|
||||
LogEvent.msg.ilike(like),
|
||||
LogEvent.event.ilike(like),
|
||||
LogEvent.request_id.ilike(like),
|
||||
LogEvent.exc.ilike(like),
|
||||
cast(LogEvent.attrs, Text).ilike(like),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def log_filters(
|
||||
levels: list[Level] = Query(default=[]),
|
||||
sources: list[str] = Query(default=[], max_length=8),
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
event: str | None = Query(
|
||||
default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$"
|
||||
),
|
||||
request_id: str | None = Query(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$"),
|
||||
job_id: int | None = Query(default=None, ge=1),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> LogFilters:
|
||||
return LogFilters(list(levels), list(sources), user, event, request_id, job_id, q, since, until)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_logs(
|
||||
filters: LogFilters = Depends(log_filters),
|
||||
before: str | None = Query(default=None, max_length=200),
|
||||
limit: int = Query(default=100, ge=1, le=PAGE_MAX),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""One page of rows, newest first, matching every filter given (they AND together); `before`
|
||||
continues from a previous page's `next_cursor`."""
|
||||
query = select(LogEvent).where(*filters.conditions())
|
||||
if before is not None:
|
||||
ts, row_id = _decode_cursor(before)
|
||||
# The `ts <=` conjunct is implied by the OR; it is there so the planner can bound the
|
||||
# `ts` index scan instead of filtering every newer row first.
|
||||
query = query.where(
|
||||
and_(
|
||||
LogEvent.ts <= ts,
|
||||
or_(LogEvent.ts < ts, and_(LogEvent.ts == ts, LogEvent.id < row_id)),
|
||||
)
|
||||
)
|
||||
|
||||
# One more than asked, to know whether there is a next page without a second count query.
|
||||
rows = (
|
||||
db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return {
|
||||
"items": [logsink.serialize(r) for r in rows],
|
||||
"accounts": _accounts(db, {r.user_id for r in rows if r.user_id is not None}),
|
||||
"next_cursor": _encode_cursor(rows[-1]) if more else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/facets")
|
||||
def log_facets(
|
||||
filters: LogFilters = Depends(log_filters),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""What the filter chips offer, with counts: the levels, sources and accounts present, and the
|
||||
most frequent events — each counted under every filter but its own (`LogFilters.conditions`).
|
||||
`total` is the "all levels" count: every filter but the level one."""
|
||||
|
||||
def counts(col, skip: Facet, cap: int | None = None) -> list[dict]:
|
||||
n = func.count().label("n")
|
||||
stmt = select(col, n).where(*filters.conditions(skip)).group_by(col).order_by(n.desc(), col)
|
||||
if cap is not None:
|
||||
stmt = stmt.limit(cap)
|
||||
return [
|
||||
{"value": NONE if value is None else value, "count": count}
|
||||
for value, count in db.execute(stmt).all()
|
||||
]
|
||||
|
||||
users = counts(LogEvent.user_id, "user")
|
||||
total = select(func.count()).select_from(LogEvent).where(*filters.conditions("levels"))
|
||||
return {
|
||||
"levels": counts(LogEvent.level, "levels"),
|
||||
"sources": counts(LogEvent.source, "sources"),
|
||||
"events": counts(LogEvent.event, "event", FACET_EVENTS_MAX),
|
||||
"users": [{**u, "value": str(u["value"])} for u in users],
|
||||
"accounts": _accounts(db, {u["value"] for u in users if u["value"] != NONE}),
|
||||
"total": db.scalar(total) or 0,
|
||||
}
|
||||
@@ -66,6 +66,12 @@ def list_audit(
|
||||
def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Clear the audit log. Leaves ONE row recording who cleared it + how many (tamper-evidence)."""
|
||||
n = db.execute(delete(AuditLog)).rowcount or 0
|
||||
audit.record(db, admin.id, AuditAction.AUDIT_CLEAR, summary=f"Cleared {n} audit entries")
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.AUDIT_CLEAR,
|
||||
summary=f"Cleared {n} audit entries",
|
||||
after={"cleared": n}, # the count as data: the log names it in the reader's language
|
||||
)
|
||||
db.commit()
|
||||
return {"cleared": n}
|
||||
return {"ok": True, "cleared": n}
|
||||
|
||||
+82
-90
@@ -3,7 +3,6 @@
|
||||
Split out of app/auth.py (R12 S2); the core auth surface (current_user, scopes, token
|
||||
helpers) stays in app/auth.py, imported below."""
|
||||
|
||||
import ipaddress
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import NamedTuple
|
||||
@@ -18,6 +17,7 @@ from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.auth import (
|
||||
@@ -39,10 +39,11 @@ from app.auth import (
|
||||
)
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import AuthToken, DemoWhitelist, Invite, OAuthToken, User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.models import AuthToken, DemoWhitelist, Invite, OAuthToken, User, display_text
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import encrypt, hash_password, hash_token, verify_password
|
||||
from app.utils import canonical_email, valid_email
|
||||
from app.refusal import refused
|
||||
|
||||
VERIFY_TTL = timedelta(hours=24)
|
||||
# A confirmation link that rides an APPROVAL or an invitation lives longer than one the person is
|
||||
@@ -172,37 +173,6 @@ def is_demo_allowed(db: Session, email: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _norm_ip(s: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
||||
"""Parse an IP for comparison, unwrapping IPv4-mapped IPv6 (`::ffff:10.0.0.1` → `10.0.0.1`) so a
|
||||
plain-IPv4 allowlist still matches a peer that surfaces in mapped form. None for a non-IP string."""
|
||||
try:
|
||||
ip = ipaddress.ip_address(s.strip())
|
||||
except ValueError:
|
||||
return None
|
||||
return ip.ipv4_mapped if getattr(ip, "ipv4_mapped", None) else ip
|
||||
|
||||
|
||||
def _client_ip(request: Request) -> str:
|
||||
"""Best-effort client IP for rate limiting. X-Forwarded-For is client-controlled, so we trust it
|
||||
ONLY when the request actually arrived from a configured reverse proxy (settings.trusted_proxy_ips
|
||||
— the address the proxy connects FROM, e.g. the VPS Caddy's WireGuard peer IP). Our proxy APPENDS
|
||||
the client it saw to XFF, so the RIGHTMOST entry is the real client and can't be forged by a
|
||||
client pre-seeding the header. Any other peer (someone hitting the app port directly, bypassing the
|
||||
proxy) is untrusted — we use its real socket IP, so it can't spoof its rate-limit identity.
|
||||
|
||||
SAFETY: this relies on `request.client` being the TRUE socket peer. uvicorn is started WITHOUT
|
||||
`--proxy-headers` (see entrypoint.sh) precisely so it never rewrites `request.client` from XFF —
|
||||
do NOT add that flag, or the trust check below would compare against a forgeable value."""
|
||||
peer = request.client.host if request.client else "unknown"
|
||||
trusted = {_norm_ip(p) for p in settings.trusted_proxy_ip_set} - {None}
|
||||
if _norm_ip(peer) in trusted:
|
||||
xff = request.headers.get("x-forwarded-for")
|
||||
parts = [p.strip() for p in (xff or "").split(",") if p.strip()]
|
||||
if parts:
|
||||
return parts[-1]
|
||||
return peer
|
||||
|
||||
|
||||
class InviteRequest(NamedTuple):
|
||||
"""What an access request settled on.
|
||||
|
||||
@@ -409,7 +379,7 @@ async def callback(
|
||||
|
||||
userinfo = token.get("userinfo")
|
||||
if not userinfo or not userinfo.get("sub"):
|
||||
raise HTTPException(status_code=400, detail="No user info returned by Google")
|
||||
raise refused(400, "google_no_userinfo")
|
||||
|
||||
sub = userinfo["sub"]
|
||||
email = (userinfo.get("email") or "").lower()
|
||||
@@ -418,7 +388,15 @@ async def callback(
|
||||
return _complete_link(request, db, link_uid, link_explicit, sub, userinfo, token)
|
||||
|
||||
if not is_allowed(db, email):
|
||||
log.warning("Login denied (not approved): %s", email or "<no email>")
|
||||
# No uid to name — having no account here is the branch. And the address is left out on
|
||||
# purpose: `redact_text` turns it into `[email]` before either sink sees it, so a line
|
||||
# whose only variable is always `[email]` says nothing. WHO asked is in the pending
|
||||
# invite recorded just below, which is where an admin looks anyway.
|
||||
log.warning(
|
||||
"Google sign-in denied: address not approved",
|
||||
# log-pii-ok: whether an address came back, not what it was
|
||||
extra=logs.event("auth.login.denied", reason="not_approved", has_email=bool(email)),
|
||||
)
|
||||
# A denied Google login doubles as an access request: record it for the admin and
|
||||
# land the user on a friendly "request received" screen instead of a raw 403.
|
||||
if email:
|
||||
@@ -451,15 +429,20 @@ async def callback(
|
||||
if existing.google_sub is not None and existing.google_sub != sub:
|
||||
# A different Google identity is already bound to this email — refuse rather than
|
||||
# hijack it. (Near-impossible with real Google accounts: email↔sub is stable.)
|
||||
log.warning("Google login email collision (different sub): %s", email)
|
||||
log.warning(
|
||||
"Google sign-in refused: a different Google identity holds this address",
|
||||
extra=logs.event("auth.login.denied", reason="sub_collision", uid=existing.id),
|
||||
)
|
||||
return RedirectResponse(url=_access_notice_url("denied"))
|
||||
if not userinfo.get("email_verified", False):
|
||||
# Adopting (and activating) a pre-existing password account requires Google to
|
||||
# actually attest the address — otherwise an unverified Google email that merely
|
||||
# collides with a pending registration could seize that account. Defense-in-depth.
|
||||
log.warning(
|
||||
"Google login: unverified email, refusing to adopt account: %s",
|
||||
email,
|
||||
"Google sign-in refused: unverified address, not adopting the account",
|
||||
extra=logs.event(
|
||||
"auth.login.denied", reason="email_unverified", uid=existing.id
|
||||
),
|
||||
)
|
||||
return RedirectResponse(url=_access_notice_url("denied"))
|
||||
user = existing
|
||||
@@ -477,7 +460,10 @@ async def callback(
|
||||
# A suspended account can't sign in by any method. Block before establishing the session and
|
||||
# tell the (verified) owner why — a successful Google auth proves they control this account.
|
||||
if user.is_suspended:
|
||||
log.info("Suspended account blocked at Google login: %s", email)
|
||||
log.info(
|
||||
"Google sign-in blocked: the account is suspended",
|
||||
extra=logs.event("auth.login.denied", reason="suspended", uid=user.id),
|
||||
)
|
||||
_notify_suspended(background, user)
|
||||
return RedirectResponse(url="/?login=suspended")
|
||||
# Record what Google actually answers with, beside the address the user registered with. The two
|
||||
@@ -486,7 +472,7 @@ async def callback(
|
||||
if email:
|
||||
user.google_email = email
|
||||
_sync_google_email(db, user, email)
|
||||
user.display_name = userinfo.get("name")
|
||||
user.display_name = display_text(userinfo.get("name"))
|
||||
user.avatar_url = userinfo.get("picture")
|
||||
# ADMIN_EMAILS (env) is the bootstrap admin list and always wins, so the configured admin can't
|
||||
# be locked out. Everyone else's role is managed in the admin UI and stored in the DB, so a
|
||||
@@ -511,17 +497,12 @@ async def callback(
|
||||
db.flush()
|
||||
|
||||
_store_token(db, user, token)
|
||||
_establish_session(request, user)
|
||||
db.commit()
|
||||
|
||||
# Multi-session: remember every account that has authenticated in this browser so the
|
||||
# user can switch between them without going through Google again. Only accounts that
|
||||
# actually completed OAuth here land in this list (it's the proof they may be switched to).
|
||||
accounts = [a for a in (request.session.get("account_ids") or []) if a != user.id]
|
||||
accounts.append(user.id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
request.session["user_id"] = user.id
|
||||
_remember_epoch(request, user)
|
||||
log.info("Login: %s (id=%s, role=%s)", email, user.id, user.role)
|
||||
log.info(
|
||||
"signed in",
|
||||
extra=logs.event("auth.login.ok", method="google", uid=user.id, role=user.role),
|
||||
)
|
||||
return RedirectResponse(url="/")
|
||||
|
||||
|
||||
@@ -613,7 +594,7 @@ def _complete_link(
|
||||
if linked_email:
|
||||
user.google_email = linked_email # see the sign-in callback: the other half of the identity
|
||||
if not user.display_name:
|
||||
user.display_name = userinfo.get("name")
|
||||
user.display_name = display_text(userinfo.get("name"))
|
||||
if not user.avatar_url:
|
||||
user.avatar_url = userinfo.get("picture")
|
||||
_store_token(db, user, token)
|
||||
@@ -631,6 +612,8 @@ async def logout(request: Request):
|
||||
out in one tab doesn't disturb another tab running a different account. If the account being
|
||||
removed was the session default, promote the most recent remaining account (or clear the
|
||||
session when none remain)."""
|
||||
# A tab still pinned to an account already signed out in another tab resolves to nobody, so it
|
||||
# signs nobody out — it used to fall back to the default and sign THAT account out in its place.
|
||||
active, is_default = resolved_user_id(request)
|
||||
remaining = [a for a in (request.session.get("account_ids") or []) if a != active]
|
||||
request.session["account_ids"] = remaining
|
||||
@@ -671,15 +654,15 @@ async def request_access(
|
||||
endpoint can't be probed for throttling either."""
|
||||
email = (body.email or "").strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
if not _request_access_limiter.allow(_client_ip(request)):
|
||||
return {"status": "pending"}
|
||||
raise refused(400, "invalid_email")
|
||||
if not _request_access_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "status": "pending"}
|
||||
if is_allowed(db, email):
|
||||
return {"status": "pending"} # already allowed — don't reveal it
|
||||
return {"ok": True, "status": "pending"} # already allowed — don't reveal it
|
||||
req = upsert_pending_invite(db, email, body.lang)
|
||||
if req.is_new:
|
||||
notify_admins_of_request(db, background, email)
|
||||
return {"status": "pending"}
|
||||
return {"ok": True, "status": "pending"}
|
||||
|
||||
|
||||
@router.post("/demo")
|
||||
@@ -689,8 +672,8 @@ def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -
|
||||
so there is no visible 'demo login' button. Rate-limited per IP; when blocked it answers
|
||||
exactly like a non-match (authenticated=false) so the field can't be hammered as an
|
||||
enumeration oracle. On a match it sets the session and the client reloads into the app."""
|
||||
if not _demo_limiter.allow(_client_ip(request)):
|
||||
return {"authenticated": False}
|
||||
if not _demo_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "authenticated": False}
|
||||
email = (body.email or "").strip().lower()
|
||||
if valid_email(email) and is_demo_allowed(db, email):
|
||||
demo = get_or_create_demo_user(db)
|
||||
@@ -699,19 +682,25 @@ def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -
|
||||
request.session.clear()
|
||||
request.session["user_id"] = demo.id
|
||||
_remember_epoch(request, demo)
|
||||
log.info("Demo login (key=%s, demo_id=%s)", email, demo.id)
|
||||
return {"authenticated": True}
|
||||
return {"authenticated": False}
|
||||
# The demo key IS an address, so it is not in the line — see the denied branch above.
|
||||
log.info("signed in", extra=logs.event("auth.login.ok", method="demo", uid=demo.id))
|
||||
return {"ok": True, "authenticated": True}
|
||||
return {"ok": True, "authenticated": False}
|
||||
|
||||
|
||||
def _establish_session(request: Request, user: User) -> None:
|
||||
"""Sign `user` in for this browser session, mirroring the Google callback's multi-account
|
||||
handling so password sign-in joins the same account switcher."""
|
||||
"""Sign `user` in for this browser session — the ONE sign-in path, shared by the Google callback
|
||||
and password login, so both join the same account switcher and both stamp `last_login_at` (the
|
||||
caller commits). Multi-session: every account that authenticated in this browser is remembered so
|
||||
the user can switch between them; only accounts that actually proved themselves HERE land in that
|
||||
wallet (it is the proof they may be switched to). The demo entry deliberately does not come
|
||||
through here: it clears the wallet instead, and its shared account has no meaningful last login."""
|
||||
accounts = [a for a in (request.session.get("account_ids") or []) if a != user.id]
|
||||
accounts.append(user.id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
request.session["user_id"] = user.id
|
||||
_remember_epoch(request, user)
|
||||
user.last_login_at = datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _app_base() -> str:
|
||||
@@ -827,23 +816,20 @@ def register(
|
||||
password = body.password or ""
|
||||
# Input validation is safe to reveal (independent of whether the email exists).
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
if len(password) < PASSWORD_MIN_LEN:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Password must be at least {PASSWORD_MIN_LEN} characters.",
|
||||
)
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
if not sysconfig.get_bool(db, "allow_registration"):
|
||||
raise HTTPException(status_code=403, detail="Registration is currently closed.")
|
||||
if not _register_limiter.allow(_client_ip(request)):
|
||||
return {"status": "ok"} # silently throttle; uniform response
|
||||
raise refused(403, "registration_closed")
|
||||
if not _register_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "status": "ok"} # silently throttle; uniform response
|
||||
|
||||
# Do the existence check + account creation OFF the response path (a background task with its own
|
||||
# session). The create path hashes the password + writes several rows + schedules emails; doing it
|
||||
# inline would make a NEW email respond measurably slower than an already-registered one (which
|
||||
# skips all that) — a timing oracle for enumeration. Off-path, any valid email responds the same.
|
||||
background.add_task(_register_account, email, password, body.lang)
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
|
||||
|
||||
def _register_account(email: str, password: str, lang: str | None = None) -> None:
|
||||
@@ -913,7 +899,12 @@ def _register_account(email: str, password: str, lang: str | None = None) -> Non
|
||||
except IntegrityError:
|
||||
# A concurrent registration for the same new email won the race between the pre-check
|
||||
# above and this flush (the unique-email constraint fires). Same already-registered
|
||||
# no-op as the pre-check hit — roll back and stop.
|
||||
# no-op as the pre-check hit — roll back and stop. The line carries nothing: the
|
||||
# answer to the caller is deliberately identical either way, and so is this.
|
||||
log.info(
|
||||
"a registration lost the race for its address",
|
||||
extra=logs.event("auth.register.race_lost"),
|
||||
)
|
||||
db.rollback()
|
||||
return
|
||||
upsert_pending_invite(db, email) # admin-approval gate
|
||||
@@ -1003,12 +994,12 @@ def verify_email_resend(
|
||||
and re-registering an already-registered email is a silent no-op, so a user who lost the first
|
||||
mail had no way back. Uniform response + timing regardless of whether the email has a pending
|
||||
account (the lookup runs off the response path), so it can't probe for registered emails."""
|
||||
if not _verify_resend_limiter.allow(_client_ip(request)):
|
||||
return {"status": "ok"} # silently throttle; uniform response
|
||||
if not _verify_resend_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "status": "ok"} # silently throttle; uniform response
|
||||
email = (body.email or "").strip().lower()
|
||||
if valid_email(email):
|
||||
background.add_task(_resend_verify_if_eligible, email)
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
|
||||
|
||||
def _resend_verify_if_eligible(email: str) -> None:
|
||||
@@ -1062,7 +1053,7 @@ def password_login(
|
||||
"""Email+password sign-in. Wrong email/password give a single uniform 401 (no enumeration).
|
||||
Once the password is proven correct, the owner gets a specific reason if their account is
|
||||
still pending or suspended — that's not an enumeration leak (they already hold the password)."""
|
||||
if not _login_limiter.allow(_client_ip(request)):
|
||||
if not _login_limiter.allow(client_key(request)):
|
||||
raise _login_refused(429, "rate_limited")
|
||||
email = (body.email or "").strip().lower()
|
||||
password = body.password or ""
|
||||
@@ -1081,7 +1072,11 @@ def password_login(
|
||||
if not user.is_active:
|
||||
raise _login_refused(403, "pending_approval", user)
|
||||
_establish_session(request, user)
|
||||
log.info("Password login: %s (id=%s, role=%s)", email, user.id, user.role)
|
||||
db.commit()
|
||||
log.info(
|
||||
"signed in",
|
||||
extra=logs.event("auth.login.ok", method="password", uid=user.id, role=user.role),
|
||||
)
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
@@ -1093,15 +1088,15 @@ def password_reset_request(
|
||||
) -> dict:
|
||||
"""Request a password-reset link. Uniform response + timing regardless of whether the email has a
|
||||
password account (the lookup runs off the response path), so it can't probe for registered emails."""
|
||||
if not _reset_limiter.allow(_client_ip(request)):
|
||||
return {"status": "ok"}
|
||||
if not _reset_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "status": "ok"}
|
||||
email = (body.email or "").strip().lower()
|
||||
# Do the account lookup + token issue + email OFF the response path (a background task with its
|
||||
# own session), so the endpoint takes the same time for any valid email whether or not it has a
|
||||
# password account — no timing-based enumeration. Any valid email schedules the same task.
|
||||
if valid_email(email):
|
||||
background.add_task(_send_reset_if_eligible, email)
|
||||
return {"status": "ok"}
|
||||
return {"ok": True, "status": "ok"}
|
||||
|
||||
|
||||
def _send_reset_if_eligible(email: str) -> None:
|
||||
@@ -1142,7 +1137,7 @@ def password_reset_check(body: TokenIn, request: Request, db: Session = Depends(
|
||||
told `true`, never `false`: this check may only ever save a user a wasted form, so when it cannot
|
||||
answer it must degrade to the old behaviour instead of declaring a live link dead.
|
||||
"""
|
||||
if not _reset_check_limiter.allow(_client_ip(request)):
|
||||
if not _reset_check_limiter.allow(client_key(request)):
|
||||
return {"ok": True}
|
||||
return {"ok": any(_peek_token(db, body.token, k) is not None for k in _PASSWORD_TOKEN_KINDS)}
|
||||
|
||||
@@ -1246,7 +1241,7 @@ def logout_others(
|
||||
are rejected, then re-stamp THIS cookie so the caller stays signed in. For "I left myself logged
|
||||
in somewhere" / a suspected session compromise. The demo account (shared) is excluded."""
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="Not available in the demo account.")
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
# Commit the bump BEFORE re-stamping this cookie, so a failed commit can't strand the current
|
||||
# session at a newer epoch than the DB (see set_password).
|
||||
bump_session_epoch(user)
|
||||
@@ -1293,16 +1288,13 @@ def set_password(
|
||||
account) needs no current password — the session already proves identity. Changing an
|
||||
existing one requires the current password, so a hijacked session can't silently rotate it."""
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="Not available in the demo account.")
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
new_password = body.password or ""
|
||||
if len(new_password) < PASSWORD_MIN_LEN:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Password must be at least {PASSWORD_MIN_LEN} characters.",
|
||||
)
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
if user.password_hash:
|
||||
if not verify_password(body.current_password or "", user.password_hash):
|
||||
raise HTTPException(status_code=403, detail="Current password is incorrect.")
|
||||
raise refused(403, "wrong_password")
|
||||
user.password_hash = hash_password(new_password)
|
||||
user.password_revoked_at = None # same as the reset path: a password exists again
|
||||
# SA4: a password change logs out every OTHER session; re-stamp this cookie at the new epoch so
|
||||
|
||||
@@ -2,15 +2,16 @@
|
||||
hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side writes
|
||||
(subscribe / unsubscribe) that require the optional write scope."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import quota
|
||||
from app.collation import collated, sorted_words
|
||||
from app.mailtext import lang_for_user
|
||||
from app import logs, quota
|
||||
from app.auth import admin_user, current_user, require_human, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -26,16 +27,17 @@ from app.models import (
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.routes._common import owned_or_404
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.sync.explore import explore_ingest_page
|
||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.sync.subscriptions import apply_channel_details, details_stale
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/channels", tags=["channels"])
|
||||
|
||||
log = logging.getLogger("siftlode.api")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
@router.get("")
|
||||
@@ -44,7 +46,7 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_
|
||||
select(Subscription, Channel)
|
||||
.join(Channel, Channel.id == Subscription.channel_id)
|
||||
.where(Subscription.user_id == user.id)
|
||||
.order_by(Subscription.priority.desc(), func.lower(Channel.title))
|
||||
.order_by(Subscription.priority.desc(), collated(Channel.title, lang_for_user(user)))
|
||||
).all()
|
||||
|
||||
# Per-channel aggregates over the shared catalog in ONE grouped pass: stored count, last
|
||||
@@ -139,7 +141,7 @@ def _discovery_rows(db: Session, user: User) -> list:
|
||||
return db.execute(
|
||||
q.group_by(Channel.id).order_by(
|
||||
func.count(func.distinct(PlaylistItem.video_id)).desc(),
|
||||
func.lower(Channel.title),
|
||||
collated(Channel.title, lang_for_user(user)),
|
||||
)
|
||||
).all()
|
||||
|
||||
@@ -166,7 +168,15 @@ def discover_channels(
|
||||
user.yt_channel_id = own
|
||||
db.commit()
|
||||
except YouTubeError as exc:
|
||||
log.warning("discovery: own-channel resolve failed (user %s): %s", user.id, exc)
|
||||
# No traceback anywhere in this file's YouTube handlers: for an upstream answer
|
||||
# `_upstream_failure` already logged the structured `youtube.api.error`, and the
|
||||
# message names the call, its status and its reason (or, with no answer at all, the
|
||||
# transport failure). The stack under it is this `except` and the client call.
|
||||
log.warning(
|
||||
"discovery: own-channel resolve failed: %s",
|
||||
exc,
|
||||
extra=logs.event("channels.discovery.own_channel_failed", reason=exc.reason),
|
||||
)
|
||||
|
||||
rows = _discovery_rows(db, user)
|
||||
|
||||
@@ -182,15 +192,23 @@ def discover_channels(
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
apply_channel_details(db, yt.get_channels(need))
|
||||
apply_channel_details(db, yt.get_channels(need), need)
|
||||
db.commit()
|
||||
# Match _discovery_rows' ORDER BY exactly: video-count DESC, then title (NULLs LAST,
|
||||
# as Postgres orders them — coercing NULL→"" would wrongly float untitled channels up).
|
||||
# Match _discovery_rows' ORDER BY exactly: video-count DESC, then title in the reader's
|
||||
# alphabet (NULLs LAST, as Postgres orders them — coercing NULL→"" would wrongly float
|
||||
# untitled channels up).
|
||||
titles = [r[0].title for r in rows if r[0].title is not None]
|
||||
alpha = {w: i for i, w in enumerate(sorted_words(db, titles, lang_for_user(user)))}
|
||||
rows = sorted(
|
||||
rows, key=lambda r: (-int(r[1]), r[0].title is None, (r[0].title or "").lower())
|
||||
rows,
|
||||
key=lambda r: (-int(r[1]), r[0].title is None, alpha.get(r[0].title or "", 0)),
|
||||
)
|
||||
except YouTubeError as exc:
|
||||
log.warning("discovery: channel enrich failed (user %s): %s", user.id, exc)
|
||||
log.warning(
|
||||
"discovery: channel enrich failed: %s",
|
||||
exc,
|
||||
extra=logs.event("channels.discovery.enrich_failed", reason=exc.reason),
|
||||
)
|
||||
db.rollback()
|
||||
|
||||
return [
|
||||
@@ -284,23 +302,30 @@ def channel_detail(
|
||||
"""One channel's "About" detail for the channel page header/About tab, plus this user's
|
||||
relationship to it (subscribed / explored) and how many of its videos are in the catalog.
|
||||
Lazily fills the About metadata (view count, join date, banner, links) the first time it's
|
||||
needed — a stub, or a channel last enriched before those fields existed (published_at is a
|
||||
reliable sentinel: every real channel has a creation date). The enrich uses the API key
|
||||
(1 unit, cached after), so it's skipped for the quota-less demo account."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
if channel.published_at is None and not user.is_demo:
|
||||
needed (a stub) and refreshes it once it is stale (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the
|
||||
subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit,
|
||||
cached after), so it's skipped for the quota-less demo account."""
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
# No `published_at is None` sentinel: a terminated channel never gets one, so it re-fetched on
|
||||
# every view; rows enriched before that field existed are long past DETAILS_MAX_AGE anyway.
|
||||
stale = details_stale(channel, datetime.now(timezone.utc))
|
||||
if stale and not user.is_demo:
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||
db.commit()
|
||||
channel = db.get(Channel, channel_id)
|
||||
except YouTubeError as exc:
|
||||
log.warning("channel detail enrich failed (%s): %s", channel_id, exc)
|
||||
log.warning(
|
||||
"channel detail enrich failed: %s",
|
||||
exc,
|
||||
extra=logs.event(
|
||||
"channels.detail.enrich_failed", channel_id=channel_id, reason=exc.reason
|
||||
),
|
||||
)
|
||||
db.rollback()
|
||||
sub = db.execute(
|
||||
select(Subscription).where(
|
||||
@@ -366,16 +391,11 @@ def explore_channel(
|
||||
it, then ingests one page of its uploads (newest-first) marked via_explore + enriched.
|
||||
Returns the next page token so the page can "load more". Guarded against the shared
|
||||
backfill quota reserve; blocked for the demo account (it spends shared quota)."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if _is_blocked(db, user, channel_id):
|
||||
raise HTTPException(status_code=403, detail="You've blocked this channel.")
|
||||
raise refused(403, "channel_blocked")
|
||||
if quota.backfill_reserve_reached():
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="The shared daily quota reserve is reached — try exploring later.",
|
||||
)
|
||||
raise refused(429, "quota_reserve")
|
||||
|
||||
# Record/refresh the exploration (DB only) before ingesting, so the explorer can see the
|
||||
# videos as soon as they land. Flag the channel ephemeral unless it's organically followed.
|
||||
@@ -401,18 +421,17 @@ def explore_channel(
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
# Enrich for the About panel + the uploads playlist id (needed to ingest videos).
|
||||
if (
|
||||
channel.details_synced_at is None
|
||||
or channel.uploads_playlist_id is None
|
||||
or channel.published_at is None
|
||||
):
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
# Stale, not "a field is missing": a terminated channel never gets an uploads playlist
|
||||
# or a creation date, and asking again on every explore would spend quota on a miss.
|
||||
if details_stale(channel, datetime.now(timezone.utc)):
|
||||
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||
db.commit()
|
||||
channel = db.get(Channel, channel_id)
|
||||
result = explore_ingest_page(db, yt, channel, page_token)
|
||||
except YouTubeError as exc:
|
||||
raise HTTPException(status_code=422, detail=f"YouTube couldn't load this channel: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
return {
|
||||
"ok": True,
|
||||
"channel_id": channel_id,
|
||||
"ingested": result["ingested"],
|
||||
"next_page_token": result["next_page_token"],
|
||||
@@ -426,7 +445,7 @@ def _user_subscription(db: Session, user: User, channel_id: str) -> Subscription
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if sub is None:
|
||||
raise HTTPException(status_code=404, detail="Not subscribed to this channel")
|
||||
raise not_found("subscription")
|
||||
return sub
|
||||
|
||||
|
||||
@@ -494,6 +513,7 @@ def update_channel(
|
||||
background.add_task(backfill_channel_now, channel_id, user.id)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"id": channel_id,
|
||||
"priority": sub.priority,
|
||||
"hidden": sub.hidden,
|
||||
@@ -512,9 +532,7 @@ def reset_backfill(
|
||||
state (a "reset" trigger). Clears the channel's backfill markers, opts it back into deep
|
||||
backfill, and re-runs its recent pull immediately; the deep scheduler re-pages the full
|
||||
back-catalog on its next run. Idempotent — videos upsert by id, so nothing duplicates."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
channel.backfill_done = False
|
||||
channel.backfill_cursor = None
|
||||
channel.recent_synced_at = None
|
||||
@@ -530,7 +548,7 @@ def reset_backfill(
|
||||
db.commit()
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
||||
run_recent_backfill(db, [channel], max_channels=1)
|
||||
return {"id": channel_id, "reset": True}
|
||||
return {"ok": True, "id": channel_id, "reset": True}
|
||||
|
||||
|
||||
@router.post("/{channel_id}/subscribe")
|
||||
@@ -544,16 +562,14 @@ def subscribe(
|
||||
local subscription with YouTube's returned resource id and enriches the channel's metadata
|
||||
if it's still a stub, so the new row renders properly in the manager. It does NOT pull the
|
||||
channel's videos — the scheduler picks those up on its next run, like any subscribed channel."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
existing = db.execute(
|
||||
select(Subscription).where(
|
||||
Subscription.user_id == user.id, Subscription.channel_id == channel_id
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise HTTPException(status_code=409, detail="Already subscribed to this channel.")
|
||||
raise refused(409, "already_subscribed")
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE),
|
||||
YouTubeClient(db, user) as yt,
|
||||
@@ -567,22 +583,31 @@ def subscribe(
|
||||
# subscription resync fills in the resource id. Any other YouTube error is a real
|
||||
# problem — surface it with a clear message (422, not a 502 that the client would
|
||||
# mistake for a transient "connection lost").
|
||||
msg = str(exc).lower()
|
||||
if "already exists" in msg or "duplicate" in msg:
|
||||
# On the reason code, never on the prose: the error no longer carries the body (it is
|
||||
# the request quoted back), and a message YouTube rewords would silently turn "already
|
||||
# following" back into a 422 the user cannot act on. `reason` is comma-joined when the
|
||||
# body listed several.
|
||||
if "subscriptionDuplicate" in exc.reason.split(","):
|
||||
yt_sub_id = ""
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"YouTube couldn't subscribe you: {exc}"
|
||||
)
|
||||
raise youtube_refusal(exc)
|
||||
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
|
||||
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
|
||||
# render. Keep the client open (we're still inside the `with`); a metadata failure here
|
||||
# must not block recording the subscription the user asked for.
|
||||
if channel.details_synced_at is None:
|
||||
try:
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
apply_channel_details(db, yt.get_channels([channel_id]), [channel_id])
|
||||
except YouTubeError as exc:
|
||||
log.warning("subscribe: stub enrich failed (%s): %s", channel_id, exc)
|
||||
log.warning(
|
||||
"subscribe: stub enrich failed: %s",
|
||||
exc,
|
||||
extra=logs.event(
|
||||
"channels.subscribe.enrich_failed",
|
||||
channel_id=channel_id,
|
||||
reason=exc.reason,
|
||||
),
|
||||
)
|
||||
db.add(
|
||||
Subscription(
|
||||
user_id=user.id,
|
||||
@@ -602,7 +627,7 @@ def subscribe(
|
||||
.values(via_explore=False)
|
||||
)
|
||||
db.commit()
|
||||
return {"subscribed": channel_id}
|
||||
return {"ok": True, "subscribed": channel_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/subscription")
|
||||
@@ -616,10 +641,7 @@ def unsubscribe(
|
||||
should Hide instead."""
|
||||
sub = _user_subscription(db, user, channel_id)
|
||||
if not sub.yt_subscription_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="No YouTube subscription id on record — sync subscriptions first.",
|
||||
)
|
||||
raise refused(400, "no_subscription_id")
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_UNSUBSCRIBE),
|
||||
@@ -629,10 +651,10 @@ def unsubscribe(
|
||||
except YouTubeError as exc:
|
||||
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
|
||||
# a transient "connection lost" gateway blip.
|
||||
raise HTTPException(status_code=422, detail=f"YouTube couldn't unsubscribe you: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
db.delete(sub)
|
||||
db.commit()
|
||||
return {"unsubscribed": channel_id}
|
||||
return {"ok": True, "unsubscribed": channel_id}
|
||||
|
||||
|
||||
@router.get("/blocked/list")
|
||||
@@ -643,7 +665,7 @@ def list_blocked(user: User = Depends(current_user), db: Session = Depends(get_d
|
||||
select(Channel)
|
||||
.join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
|
||||
.where(BlockedChannel.user_id == user.id)
|
||||
.order_by(func.lower(Channel.title))
|
||||
.order_by(collated(Channel.title, lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -663,8 +685,7 @@ def block_channel(
|
||||
"""Block a channel for this user: its videos won't appear in their live search (and won't be
|
||||
ingested for them) and are hidden from their feed / explore. Idempotent. The now-hidden
|
||||
un-kept search/explore videos are reclaimed by the discovery-cleanup job in due course."""
|
||||
if db.get(Channel, channel_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
get_or_404(db, Channel, channel_id, what="channel")
|
||||
if not _is_blocked(db, user, channel_id):
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=channel_id))
|
||||
# Stop any active exploration of it so it can be cleaned up.
|
||||
@@ -674,7 +695,7 @@ def block_channel(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"blocked": channel_id}
|
||||
return {"ok": True, "blocked": channel_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/block")
|
||||
@@ -689,7 +710,7 @@ def unblock_channel(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"unblocked": channel_id}
|
||||
return {"ok": True, "unblocked": channel_id}
|
||||
|
||||
|
||||
class AttachTagIn(BaseModel):
|
||||
@@ -706,7 +727,7 @@ def attach_tag(
|
||||
_user_subscription(db, user, channel_id)
|
||||
# Only the user's own tags may be attached as personal links (owned_or_404 → 404 if the tag
|
||||
# is missing OR belongs to someone else, without leaking which).
|
||||
owned_or_404(db, Tag, body.tag_id, user, detail="Unknown tag")
|
||||
owned_or_404(db, Tag, body.tag_id, user, what="tag")
|
||||
exists = db.execute(
|
||||
select(ChannelTag).where(
|
||||
ChannelTag.channel_id == channel_id,
|
||||
@@ -724,7 +745,7 @@ def attach_tag(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"channel_id": channel_id, "tag_id": body.tag_id}
|
||||
return {"ok": True, "channel_id": channel_id, "tag_id": body.tag_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/tags/{tag_id}")
|
||||
@@ -744,4 +765,4 @@ def detach_tag(
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"channel_id": channel_id, "tag_id": tag_id}
|
||||
return {"ok": True, "channel_id": channel_id, "tag_id": tag_id}
|
||||
|
||||
@@ -4,18 +4,23 @@ app.sysconfig. Secret values (e.g. SMTP password) are write-only — never retur
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import User
|
||||
from app.refusal import refused
|
||||
from app.issues import gitea
|
||||
from app.routes._common import issue_refused, not_found
|
||||
from app.refusal import setting_refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin/config", tags=["admin"])
|
||||
|
||||
@@ -29,6 +34,14 @@ def _redact(v):
|
||||
return _URL_USERINFO.sub(r"\1***@", v) if isinstance(v, str) else v
|
||||
|
||||
|
||||
def _apply_runtime(db: Session, key: str) -> None:
|
||||
"""Some knobs change how THIS process behaves, not just what a later read returns — they have to
|
||||
take effect now, without a restart. One dispatch for both call sites (set and reset), so the next
|
||||
such key can't be wired into one of them and forgotten in the other."""
|
||||
if key == "log_level":
|
||||
logs.set_level(sysconfig.get_str(db, "log_level"))
|
||||
|
||||
|
||||
@router.get("")
|
||||
def get_config(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
return sysconfig.describe(db)
|
||||
@@ -50,19 +63,16 @@ def set_config(
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown config key")
|
||||
raise not_found("config_key")
|
||||
if "value" not in body.model_fields_set:
|
||||
raise HTTPException(status_code=400, detail="Missing 'value'")
|
||||
raise refused(400, "value_missing")
|
||||
if s.secret and not sysconfig.secrets_manageable():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Set TOKEN_ENCRYPTION_KEY to store secrets in the database.",
|
||||
)
|
||||
raise refused(400, "no_encryption_key")
|
||||
old = None if s.secret else sysconfig.get(db, key)
|
||||
try:
|
||||
sysconfig.set_value(db, key, body.value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc) or "Invalid value")
|
||||
except sysconfig.InvalidSetting as exc:
|
||||
raise setting_refused(exc) from None
|
||||
# Never log a secret's value — only that it was changed, by whom. Non-secret values are logged
|
||||
# but with URL credentials redacted (a proxy/URL setting can embed user:pass). Skip the row when
|
||||
# a non-secret value didn't actually change (a no-op re-save shouldn't add trail noise); secrets
|
||||
@@ -80,6 +90,7 @@ def set_config(
|
||||
after=None if s.secret else {"value": _redact(new)},
|
||||
)
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
|
||||
@@ -91,7 +102,7 @@ def reset_config(
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown config key")
|
||||
raise not_found("config_key")
|
||||
old = None if s.secret else sysconfig.get(db, key)
|
||||
sysconfig.reset(db, key)
|
||||
audit.record(
|
||||
@@ -104,6 +115,7 @@ def reset_config(
|
||||
before=None if s.secret else {"value": _redact(old)},
|
||||
)
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
|
||||
@@ -115,8 +127,21 @@ def send_test_email(
|
||||
"""Send a test email to the requesting admin using the current (DB or env) SMTP config —
|
||||
lets the admin verify the settings actually work."""
|
||||
if not email_mod.email_enabled():
|
||||
raise HTTPException(status_code=400, detail="SMTP is not configured.")
|
||||
raise refused(400, "smtp_off")
|
||||
sent = email_mod.send_test(user.email, mailtext.lang_for_user(user))
|
||||
if not sent:
|
||||
raise HTTPException(status_code=422, detail="SMTP send failed — check the settings.")
|
||||
return {"sent": True, "to": user.email}
|
||||
raise refused(422, "smtp_failed")
|
||||
return {"ok": True, "sent": True, "to": user.email}
|
||||
|
||||
|
||||
@router.post("/test-issues")
|
||||
def test_issue_tracker(
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Check the configured issue tracker (E16): the repository answers with this token, takes
|
||||
issues, and is PRIVATE — the one it must be, since an issue carries its reporter's log."""
|
||||
try:
|
||||
return {"ok": True, **gitea.check(db)}
|
||||
except gitea.GiteaError as exc:
|
||||
raise issue_refused(exc) from None
|
||||
|
||||
@@ -0,0 +1,418 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, the screenshots, and the evidence under review.
|
||||
|
||||
The session itself — what tags a request, its cap, its level — is `app/debugsession.py`. These routes
|
||||
are the user's handle on their OWN sessions only: another user's id 404s (`owned_or_404`), including
|
||||
for an admin, who reads everyone's lines in the diagnostics console instead. The shared demo account
|
||||
cannot record: its log would interleave every visitor under one account.
|
||||
|
||||
Screenshots arrive as a raw image body (`image/png`, `image/jpeg`, `image/webp`), not a multipart
|
||||
form — one image per request, capped while it arrives, and kept only if its bytes really are one of
|
||||
those types, whatever the Content-Type claimed.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, debugsession, logs, logsink
|
||||
from app.audit import AuditAction
|
||||
from app.auth import require_human
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.issues import bundle, gitea
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.refusal import refused
|
||||
from app.models import DebugScreenshot, DebugSession, User
|
||||
from app.routes._common import capped_body, issue_refused, not_found, owned_or_404
|
||||
from app.utils import iso
|
||||
|
||||
router = APIRouter(prefix="/api/debug-sessions", tags=["diagnostics"])
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Sessions listed in Settings → Diagnosztika, newest first: enough to reach last week's unsent draft.
|
||||
LIST_MAX = 20
|
||||
|
||||
|
||||
class StartIn(BaseModel):
|
||||
level: debugsession.SessionLevel
|
||||
|
||||
|
||||
_view = debugsession.view
|
||||
|
||||
# New sessions a user may open per hour (joining the running one is not counted): a session is a
|
||||
# place to keep screenshots, and this — with the per-user storage cap — bounds how many there are.
|
||||
_new_sessions = RateLimiter(max_events=20, window_seconds=3600)
|
||||
|
||||
|
||||
@router.post("")
|
||||
def start_session(
|
||||
body: StartIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Start recording, or join the recording already running (another tab started it)."""
|
||||
joining = debugsession.open_session(db, user) is not None
|
||||
if not joining and not _new_sessions.allow(str(user.id)):
|
||||
raise refused(429, "debug_session_rate_limited", limit=_new_sessions.max_events)
|
||||
try:
|
||||
session = debugsession.start(db, user, body.level)
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
# A concurrent Start won the one-open-per-user index: join that one.
|
||||
db.rollback()
|
||||
session = debugsession.open_session(db, user)
|
||||
if session is None:
|
||||
raise refused(409, "debug_session_conflict") from None
|
||||
# The rest of THIS request is part of the session too.
|
||||
level, until = debugsession.level_for(user, session)
|
||||
logs.set_user(user.id, level, until, session.id)
|
||||
log.info(
|
||||
"debug session started",
|
||||
extra=logs.event("debug.session.started", level=session.level, until=iso(session.until)),
|
||||
)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.post("/{sid}/stop")
|
||||
def stop_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if session.ended_at is None:
|
||||
# Logged BEFORE the session closes, while this request still carries its tag: the last line
|
||||
# of the evidence is the Stop itself.
|
||||
log.info("debug session stopped", extra=logs.event("debug.session.stopped"))
|
||||
debugsession.stop(session)
|
||||
db.commit()
|
||||
logs.end_session_elevation(session.id)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_sessions(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
sessions = db.scalars(
|
||||
select(DebugSession)
|
||||
.where(DebugSession.user_id == user.id)
|
||||
.order_by(DebugSession.started_at.desc())
|
||||
.limit(LIST_MAX)
|
||||
).all()
|
||||
return {"items": [_view(db, s) for s in sessions]}
|
||||
|
||||
|
||||
@router.get("/{sid}")
|
||||
def get_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
return _view(db, owned_or_404(db, DebugSession, sid, user, what="debug_session"))
|
||||
|
||||
|
||||
@router.get("/{sid}/events")
|
||||
def session_events(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""The rows an issue from this session would carry — browser and server, in time order."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
rows, total = debugsession.bundle(db, session)
|
||||
return {"items": [logsink.serialize(r) for r in rows], "total": total}
|
||||
|
||||
|
||||
# --- the draft and its export ---------------------------------------------------------------
|
||||
# The composer's own fields (steps, description…) are its business, kept whole so it reopens as it
|
||||
# was — bounded here so a draft cannot become a place to store anything else.
|
||||
FORM_MAX_BYTES = 64 * 1024
|
||||
# Zips a user may build per minute: each holds every screenshot in memory while it is built.
|
||||
_exports = RateLimiter(max_events=6, window_seconds=60)
|
||||
|
||||
|
||||
class DraftIn(BaseModel):
|
||||
title: str = Field(max_length=200)
|
||||
# The rendered markdown the export and the issue carry.
|
||||
body: str = Field(max_length=50_000)
|
||||
# Log rows taken out of the evidence in the review.
|
||||
excluded: list[int] = Field(default_factory=list, max_length=debugsession.BUNDLE_MAX)
|
||||
form: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@router.put("/{sid}/draft")
|
||||
def save_draft(
|
||||
sid: str,
|
||||
body: DraftIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if len(json.dumps(body.form, ensure_ascii=False).encode()) > FORM_MAX_BYTES:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
# Under the lock, and refused while a send is under way: the issue is being built from the draft
|
||||
# as it was saved, and this would change it under the send.
|
||||
debugsession.lock(db, session)
|
||||
if debugsession.sending(session):
|
||||
raise refused(409, "issue_sending")
|
||||
if session.issue_url is not None:
|
||||
# Sent: the issue is on the tracker now, and a draft edited after it would say otherwise.
|
||||
raise refused(409, "issue_already_sent")
|
||||
debugsession.save_draft(session, body.model_dump())
|
||||
db.commit()
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.get("/{sid}/export")
|
||||
def export_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
"""The issue as a zip — the text, the two logs and the screenshots — for an instance without an
|
||||
issue tracker, or for sending it by hand. Built from the SAVED draft: what the composer shows is
|
||||
what it saved."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if not _exports.allow(str(user.id)):
|
||||
raise refused(429, "export_rate_limited")
|
||||
data = bundle.zipped(bundle.files(db, session))
|
||||
name = f"siftlode-debug-{session.started_at:%Y%m%d-%H%M}-{session.id[:6]}.zip"
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="application/zip",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{name}"',
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# --- the issue tracker (S4) -----------------------------------------------------------------
|
||||
# Issues a user may open per hour — counted once the tracker is ASKED to open one (an answer that
|
||||
# goes missing may still have created it, so an attempt cannot be given back); finishing a half-sent
|
||||
# one, or an attempt refused before the tracker was asked, is not counted. The composer sends one at
|
||||
# a time; this is room for a bad afternoon, not a pace anything legitimate keeps up.
|
||||
_new_issues = RateLimiter(max_events=5, window_seconds=3600)
|
||||
# Attempts on the tracker per hour, creating or finishing. Deliberately NOT the budget above: a
|
||||
# half-sent issue must always be finishable, so the pace that guards creating one must never be what
|
||||
# stands between a user and their own report. This one only bounds a loop — every attempt re-reads
|
||||
# the whole bundle into memory and re-offers the pending file to the tracker, and a retry that is
|
||||
# refused for good (the tracker's own limits) never leaves `pending`. No hand reaches it.
|
||||
_sends = RateLimiter(max_events=60, window_seconds=3600)
|
||||
# The composer's live preview, rendered BY the tracker: one authenticated call each, with our token.
|
||||
# The composer's toggle fires it once per switch INTO preview — the editor is the other branch, so
|
||||
# no keystroke can change the query key while the preview is up — and react-query's cache makes an
|
||||
# unchanged draft free. (It does NOT debounce, whatever this comment used to say: a key that changes
|
||||
# per keystroke is one fetch per keystroke, measured. Showing the preview BESIDE the fields would
|
||||
# need a real debounce first.) This bounds what a loop can aim at the tracker.
|
||||
_previews = RateLimiter(max_events=60, window_seconds=60)
|
||||
|
||||
|
||||
def _issues_rate_limited() -> HTTPException:
|
||||
return refused(429, "issues_rate_limited", limit=_new_issues.max_events)
|
||||
|
||||
|
||||
@router.post("/{sid}/issue")
|
||||
def send_issue(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Send the session's issue to the tracker — from the SAVED draft, as the export is built — or
|
||||
finish a send that stopped half-way (the issue exists; only what is missing is sent)."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
# The session is CLAIMED for the send rather than locked through it: the tracker is slow (a
|
||||
# dozen calls, 20 s each), and a row lock held across them would queue this user's Stop, their
|
||||
# autosave and their screenshots behind it. They see the claim instead and are refused at once.
|
||||
debugsession.lock(db, session)
|
||||
if debugsession.sending(session):
|
||||
raise refused(409, "issue_sending")
|
||||
new = session.issue_url is None
|
||||
if not new and not debugsession.unfinished(session):
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
key = str(user.id)
|
||||
if new:
|
||||
if not ((session.draft or {}).get("title") or "").strip():
|
||||
raise refused(422, "issue_title_required")
|
||||
if _new_issues.exhausted(key):
|
||||
raise _issues_rate_limited()
|
||||
if not _sends.allow(key):
|
||||
raise refused(429, "issue_rate_limited", limit=_sends.max_events)
|
||||
token = debugsession.claim_send(session)
|
||||
db.commit() # the claim is what everything else sees; the lock ends here
|
||||
|
||||
def count_new_issue() -> None:
|
||||
if not _new_issues.allow(key):
|
||||
raise _issues_rate_limited()
|
||||
|
||||
try:
|
||||
gitea.deliver(
|
||||
db,
|
||||
session,
|
||||
token=token,
|
||||
checkpoint=db.commit,
|
||||
before_create=count_new_issue,
|
||||
on_created=lambda: _record_created(db, user, session),
|
||||
)
|
||||
except debugsession.SendClaimLost:
|
||||
# This send outlived its lease and another one took the session over while it was still
|
||||
# talking to the tracker. It stopped rather than writing over that one's progress — and a
|
||||
# send IS under way, just not ours, which is what the caller needs to hear.
|
||||
raise refused(409, "issue_sending") from None
|
||||
except gitea.GiteaError as exc:
|
||||
# Keep what did happen: an issue created before an upload failed stays linked, and the
|
||||
# retry does only what is still missing.
|
||||
raise issue_refused(exc) from None
|
||||
except HTTPException:
|
||||
raise # the pace, refused before the tracker was asked
|
||||
finally:
|
||||
_end_send(db, session, token)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
def _end_send(db: Session, session: DebugSession, token: str) -> None:
|
||||
"""Give the claim back, whatever else happened. A failure here must not replace the failure that
|
||||
brought us: a dead transaction (the database went away mid-send) is rolled back, and the claim
|
||||
then simply ages out."""
|
||||
try:
|
||||
debugsession.lock(db, session)
|
||||
debugsession.release_send(session, token)
|
||||
db.commit()
|
||||
except Exception: # noqa: BLE001 — whatever went wrong here, the send's own failure is the news
|
||||
db.rollback()
|
||||
log.warning(
|
||||
"a send could not give its claim back",
|
||||
extra=logs.event("issues.claim.stuck", session_id=session.id),
|
||||
)
|
||||
|
||||
|
||||
def _record_created(db: Session, user: User, session: DebugSession) -> None:
|
||||
"""The trail for the one act that takes a user's log off this instance. Staged inside the send,
|
||||
so the checkpoint that first writes `issue_url` commits BOTH — recorded later it could be lost
|
||||
on its own (the process dies mid-send), and no retry could write it afterwards: the next send
|
||||
sees the link and knows the issue is not new."""
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
AuditAction.ISSUE_CREATE,
|
||||
target_type="debug_session",
|
||||
target_id=session.id,
|
||||
summary=session.issue_url or "",
|
||||
)
|
||||
|
||||
|
||||
class PreviewIn(BaseModel):
|
||||
text: str = Field(max_length=50_000)
|
||||
|
||||
|
||||
@router.post("/{sid}/preview")
|
||||
def preview_issue(
|
||||
sid: str,
|
||||
body: PreviewIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The body as the tracker will render it. Gitea sanitises its own output; the composer still
|
||||
shows it in a sandboxed frame, never in the app's own document."""
|
||||
owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if not _previews.allow(str(user.id)):
|
||||
raise refused(429, "preview_rate_limited")
|
||||
try:
|
||||
return {"ok": True, "html": gitea.preview(db, body.text)}
|
||||
except gitea.GiteaError as exc:
|
||||
raise issue_refused(exc) from None
|
||||
|
||||
|
||||
# --- screenshots ---------------------------------------------------------------------------
|
||||
def _shot_view(shot: DebugScreenshot) -> dict:
|
||||
return {"id": shot.id, "mime": shot.mime, "size": shot.size, "created_at": iso(shot.created_at)}
|
||||
|
||||
|
||||
@router.get("/{sid}/screenshots")
|
||||
def list_screenshots(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
return {"items": [_shot_view(s) for s in debugsession.screenshots(db, session)]}
|
||||
|
||||
|
||||
def _upload_target(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> DebugSession:
|
||||
return owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
|
||||
|
||||
async def _image_body(request: Request, db: Session = Depends(get_db)) -> bytes:
|
||||
# The owner check above has already run, and it was a SELECT — so this session is holding a
|
||||
# pooled connection `idle in transaction` and would hold it for the whole 10 MiB upload, from a
|
||||
# client that may be on a phone. Hand it back first; the handler's own work re-takes one. The
|
||||
# `session` object the handler holds is expired by this and reloads on its next attribute read,
|
||||
# which is the correct moment to touch the database again.
|
||||
release_read_connection(db)
|
||||
return await capped_body(request, debugsession.SCREENSHOT_MAX_BYTES)
|
||||
|
||||
|
||||
@router.post("/{sid}/screenshots")
|
||||
def add_screenshot(
|
||||
# The owner check FIRST: dependencies resolve in order, so a stranger's upload is refused
|
||||
# before a byte of it is read.
|
||||
session: DebugSession = Depends(_upload_target),
|
||||
data: bytes = Depends(_image_body),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
try:
|
||||
shot = debugsession.add_screenshot(db, session, data)
|
||||
except debugsession.ScreenshotRefused as exc:
|
||||
raise _screenshot_refused(exc.code) from None
|
||||
db.commit()
|
||||
return {"ok": True, "screenshot": _shot_view(shot)}
|
||||
|
||||
|
||||
def _screenshot_refused(code: str) -> HTTPException:
|
||||
# Literal reasons, one per case: the frontend's contract test reads them to require a sentence
|
||||
# for each in both languages.
|
||||
if code == "already_sent":
|
||||
return refused(409, "screenshot_already_sent")
|
||||
if code == "too_many":
|
||||
return refused(422, "screenshot_too_many", limit=debugsession.SCREENSHOTS_MAX)
|
||||
if code == "sending":
|
||||
return refused(409, "issue_sending")
|
||||
if code == "storage_full":
|
||||
mb = debugsession.SCREENSHOT_USER_MAX_BYTES // (1024 * 1024)
|
||||
return refused(422, "screenshot_storage_full", limit=mb)
|
||||
if code == "empty":
|
||||
return refused(422, "screenshot_empty")
|
||||
return refused(422, "screenshot_not_an_image")
|
||||
|
||||
|
||||
def _session_screenshot(db: Session, sid: str, shot_id: int, user: User) -> DebugScreenshot:
|
||||
return _screenshot_of(
|
||||
db, owned_or_404(db, DebugSession, sid, user, what="debug_session"), shot_id
|
||||
)
|
||||
|
||||
|
||||
def _screenshot_of(db: Session, session: DebugSession, shot_id: int) -> DebugScreenshot:
|
||||
shot = db.get(DebugScreenshot, shot_id)
|
||||
if shot is None or shot.session_id != session.id:
|
||||
raise not_found("screenshot")
|
||||
return shot
|
||||
|
||||
|
||||
@router.get("/{sid}/screenshots/{shot_id}")
|
||||
def get_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
shot = _session_screenshot(db, sid, shot_id, user)
|
||||
# `nosniff`: the type was checked against the bytes at upload, and the browser must not guess.
|
||||
return Response(
|
||||
content=shot.data,
|
||||
media_type=shot.mime,
|
||||
headers={"Cache-Control": "private, max-age=3600", "X-Content-Type-Options": "nosniff"},
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{sid}/screenshots/{shot_id}")
|
||||
def delete_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
shot = _screenshot_of(db, session, shot_id)
|
||||
try:
|
||||
debugsession.remove_screenshot(db, session, shot)
|
||||
except debugsession.ScreenshotRefused as exc:
|
||||
raise _screenshot_refused(exc.code) from None
|
||||
db.commit()
|
||||
return {"ok": True, "deleted": shot_id}
|
||||
@@ -0,0 +1,190 @@
|
||||
"""The browser's log, sent while its user has a raised diagnostics level (E22 S5).
|
||||
|
||||
The client keeps its log in memory (frontend `lib/log.ts`). Only after the user consented to Részletes
|
||||
or Teljes in Settings → Diagnosztika does it stream that log here, in batches, and each batch lands in
|
||||
`log_events` beside the server's own rows: `source="web"`, the account from the SESSION (never from
|
||||
the body), and the request id each event carries, which is what joins a browser line to the server
|
||||
line of the same request.
|
||||
|
||||
Nothing from the body is trusted as-is:
|
||||
* only a JSON Content-Type is read, and the whole body is capped before it is parsed, and so are
|
||||
the number of events and each event's attributes;
|
||||
* the pace is capped per account, and so is what an account may store per hour;
|
||||
* everything is redacted AGAIN with the server's own rules — the client redacts at capture, but a
|
||||
modified client would not;
|
||||
* an event below the consented level is dropped (a Részletes consent does not store what only
|
||||
Teljes may carry), and so is the whole batch once the level has lapsed — where an open debug
|
||||
session the request names counts as consent to ITS level (E16), and only its own events keep
|
||||
its tag;
|
||||
* the timestamp is clamped to the consent window, so a row cannot dodge the retention job by
|
||||
claiming to come from the future.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy import insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, diagnostics, logs, logsink
|
||||
from app.auth import current_user
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.models import DebugSession, LogEvent, User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import capped_body
|
||||
|
||||
router = APIRouter(prefix="/api/diagnostics", tags=["diagnostics"])
|
||||
|
||||
BODY_MAX_BYTES = 512 * 1024
|
||||
EVENTS_MAX = 200
|
||||
ATTRS_MAX_BYTES = 16 * 1024
|
||||
# The client flushes every few seconds and when its buffer fills; this is room for a burst, not a
|
||||
# pace anything legitimate keeps up for a minute.
|
||||
_batches = RateLimiter(max_events=30, window_seconds=60)
|
||||
# What one account may STORE per hour. The pace cap alone still lets a self-consented account write
|
||||
# ~15 MiB a minute into the one shared database, kept for the whole retention window; a real session
|
||||
# at Teljes stores a few hundred kB an hour. Past the budget a batch is accepted and not stored (a
|
||||
# refusal would only make the client resend it), and one notice per hour says so.
|
||||
STORED_BYTES_PER_HOUR = 5 * 1024 * 1024
|
||||
_stored_bytes = RateLimiter(max_events=STORED_BYTES_PER_HOUR, window_seconds=3600)
|
||||
_budget_notices = RateLimiter(max_events=1, window_seconds=3600)
|
||||
|
||||
ClientLevel = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
|
||||
|
||||
class ClientEvent(BaseModel):
|
||||
ts: datetime
|
||||
level: ClientLevel
|
||||
event: str = Field(max_length=64, pattern=r"^[a-z][a-z0-9_]*(\.[a-z0-9_]+)*$")
|
||||
msg: str = Field(max_length=8_000)
|
||||
attrs: dict[str, Any] | None = None
|
||||
request_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$")
|
||||
tab_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,32}$")
|
||||
# The debug session the event was captured in (E16); kept only when it is the one this request
|
||||
# names in `X-Debug-Session` and the auth dependency found it the caller's own and open.
|
||||
debug_session_id: str | None = Field(default=None, pattern=r"^[0-9a-f]{16}$")
|
||||
version: str | None = Field(default=None, max_length=64)
|
||||
|
||||
|
||||
class ClientEventsIn(BaseModel):
|
||||
events: list[ClientEvent] = Field(max_length=EVENTS_MAX)
|
||||
|
||||
|
||||
async def _parsed_events(request: Request, db: Session = Depends(get_db)) -> ClientEventsIn:
|
||||
"""The body as the model — parsed only after the size cap, which is why this is a dependency and
|
||||
not a plain body parameter (FastAPI reads a declared body in full before anything else runs).
|
||||
|
||||
JSON only, by its Content-Type, as a declared body would be. Parsing any body as JSON would let a
|
||||
CORS-simple `text/plain` form post from a same-site page write events into a signed-in user's
|
||||
log; the app's own client always sends `application/json`, the unload beacon included."""
|
||||
media_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media_type != "application/json" and not media_type.endswith("+json"):
|
||||
raise HTTPException(status_code=415, detail={"code": "json_required"})
|
||||
# `current_user` resolved above on a SELECT, so this session is holding a pooled connection.
|
||||
# Give it back before awaiting the body — the sink is a background beacon and its sender may be
|
||||
# on a link far slower than the 512 KiB suggests. See `db.release_read_connection`.
|
||||
release_read_connection(db)
|
||||
raw = await capped_body(request, BODY_MAX_BYTES)
|
||||
try:
|
||||
return ClientEventsIn.model_validate_json(raw)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=422, detail=json.loads(exc.json())) from exc
|
||||
|
||||
|
||||
def _row(
|
||||
item: ClientEvent,
|
||||
user: User,
|
||||
now: datetime,
|
||||
window_start: datetime,
|
||||
session: DebugSession | None,
|
||||
) -> dict[str, Any]:
|
||||
attrs = logs.redact(item.attrs) if item.attrs else None
|
||||
if attrs is not None:
|
||||
size = len(json.dumps(attrs, ensure_ascii=False, default=str).encode())
|
||||
if size > ATTRS_MAX_BYTES:
|
||||
attrs = {"attrs_dropped_bytes": size}
|
||||
if item.tab_id:
|
||||
attrs = {**(attrs or {}), "tab_id": item.tab_id}
|
||||
ts = item.ts if item.ts.tzinfo else item.ts.replace(tzinfo=timezone.utc)
|
||||
return logsink.to_row(
|
||||
{
|
||||
"ts": min(max(ts, window_start), now),
|
||||
"level": item.level,
|
||||
"sev": logs.SEVERITY[item.level],
|
||||
"source": "web",
|
||||
"logger": "web",
|
||||
"event": item.event,
|
||||
"msg": logs.redact_text(item.msg),
|
||||
"attrs": attrs,
|
||||
"request_id": item.request_id,
|
||||
"user_id": user.id,
|
||||
"debug_session_id": (
|
||||
session.id if session and item.debug_session_id == session.id else None
|
||||
),
|
||||
"version": item.version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/events")
|
||||
def ingest_events(
|
||||
# The account first: dependencies resolve in order, so an anonymous caller is refused before a
|
||||
# byte of its body is read.
|
||||
request: Request,
|
||||
user: User = Depends(current_user),
|
||||
body: ClientEventsIn = Depends(_parsed_events),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# A debug session the request names raises the level for its own events (E16): starting one is
|
||||
# the consent to that level for its duration, whatever the user's own level is.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, _ = debugsession.level_for(user, session)
|
||||
if level == diagnostics.BASE:
|
||||
# Lapsed or withdrawn: the client stops streaming when it sees this.
|
||||
raise HTTPException(status_code=409, detail={"code": "diagnostics_off"})
|
||||
if not _batches.allow(str(user.id)):
|
||||
raise HTTPException(status_code=429, detail={"code": "rate_limited"})
|
||||
now = datetime.now(timezone.utc)
|
||||
window_start = now - timedelta(hours=diagnostics.ELEVATION_HOURS)
|
||||
floor = logs.SEVERITY[level]
|
||||
rows = [
|
||||
_row(item, user, now, window_start, session)
|
||||
for item in body.events
|
||||
if logs.SEVERITY[item.level] >= floor
|
||||
]
|
||||
if not rows:
|
||||
return {"ok": True, "stored": 0}
|
||||
size = sum(len(json.dumps(row, ensure_ascii=False, default=str).encode()) for row in rows)
|
||||
stored = len(rows)
|
||||
if not _stored_bytes.allow(str(user.id), size):
|
||||
stored = 0
|
||||
if not _budget_notices.allow(str(user.id)):
|
||||
return {"ok": True, "stored": 0}
|
||||
rows = [_budget_notice(user, now)]
|
||||
# Under the erasure guard until the commit: an account erased meanwhile gets no rows back.
|
||||
with logs.erasure_guard():
|
||||
db.execute(insert(LogEvent), logsink.without_erased(rows))
|
||||
db.commit()
|
||||
return {"ok": True, "stored": stored}
|
||||
|
||||
|
||||
def _budget_notice(user: User, now: datetime) -> dict[str, Any]:
|
||||
return logsink.to_row(
|
||||
{
|
||||
"ts": now,
|
||||
"level": "WARNING",
|
||||
"sev": logs.SEVERITY["WARNING"],
|
||||
"source": "web",
|
||||
"logger": "web",
|
||||
"event": "diagnostics.budget_spent",
|
||||
# What the budget does, exactly: it is a SLIDING hour, and a refused batch is not
|
||||
# charged to it, so smaller batches keep landing while there is room for them. Saying
|
||||
# "the rest of the hour is not kept" would be a row contradicting the rows beneath it.
|
||||
"msg": "The browser log passed its hourly storage budget; a batch bigger than what is left of it is not kept",
|
||||
"attrs": {"budget_bytes": STORED_BYTES_PER_HOUR},
|
||||
"user_id": user.id,
|
||||
}
|
||||
)
|
||||
@@ -11,9 +11,9 @@ thing it exists to remove.
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -23,7 +23,8 @@ from app.discover import hero as hero_mod
|
||||
from app.discover import pins, rails
|
||||
from app.discover.budget import Budget, DEFAULT_BUDGET
|
||||
from app.models import ComfortPin, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.plex.cards import norm_lang
|
||||
from app.routes._common import owned_or_404, not_found
|
||||
|
||||
router = APIRouter(prefix="/api/discover", tags=["discover"])
|
||||
|
||||
@@ -45,6 +46,7 @@ class ReorderIn(BaseModel):
|
||||
def get_discover(
|
||||
budget: Budget = DEFAULT_BUDGET,
|
||||
seed: int = 0,
|
||||
lang: str = "en",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -52,11 +54,14 @@ def get_discover(
|
||||
|
||||
`resume` and `shelf` are built once and handed to the hero, so the two rails the hero draws from
|
||||
are not queried twice for a single page load."""
|
||||
resume = rails.continue_watching(db, user)
|
||||
shelf = pins.shelf(db, user)
|
||||
lang = norm_lang(lang)
|
||||
resume = rails.continue_watching(db, user, lang=lang)
|
||||
# Plex off → the shelf rail (and so the hero's comfort tier) goes YouTube-only like the rails
|
||||
# beside it; the pins are kept and `GET /pins` still lists them (`pins.shelf`).
|
||||
shelf = pins.shelf(db, user, include_plex=rails.plex_available(db), lang=lang)
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf),
|
||||
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf, lang=lang),
|
||||
"continue_watching": resume,
|
||||
"comfort": shelf,
|
||||
}
|
||||
@@ -78,6 +83,7 @@ def get_pick(
|
||||
seed: int = 0,
|
||||
exclude: str = "",
|
||||
start: int = 0,
|
||||
lang: str = "en",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -97,18 +103,25 @@ def get_pick(
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(
|
||||
db, user, budget, seed, exclude=set(list(skip)[:MAX_EXCLUDE]), start=start
|
||||
db,
|
||||
user,
|
||||
budget,
|
||||
seed,
|
||||
exclude=set(list(skip)[:MAX_EXCLUDE]),
|
||||
start=start,
|
||||
lang=norm_lang(lang),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/pins")
|
||||
def list_pins(
|
||||
lang: str = "en",
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The shelf on its own — for the shelf editor, which needs it without the rest of the page."""
|
||||
return {"items": pins.shelf(db, user)}
|
||||
return {"items": pins.shelf(db, user, lang=norm_lang(lang))}
|
||||
|
||||
|
||||
@router.post("/pins")
|
||||
@@ -125,7 +138,7 @@ def add_pin(
|
||||
constraint is the mechanism, not the message."""
|
||||
resolved = pins.resolve_ref(db, user, body.kind, body.ref)
|
||||
if resolved is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
raise not_found("item")
|
||||
column, value = resolved
|
||||
|
||||
existing = db.execute(
|
||||
@@ -157,7 +170,7 @@ def add_pin(
|
||||
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
||||
# item), which trips the FK. Semantically that is the same "no such thing" the pre-check
|
||||
# returns above — re-raising handed the user a 500 they could do nothing with.
|
||||
raise HTTPException(status_code=404, detail="Unknown item")
|
||||
raise not_found("item")
|
||||
return {"ok": True, "id": winner.id, "created": False}
|
||||
return {"ok": True, "id": pin.id, "created": True}
|
||||
|
||||
@@ -168,7 +181,7 @@ def remove_pin(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pin = owned_or_404(db, ComfortPin, pin_id, user)
|
||||
pin = owned_or_404(db, ComfortPin, pin_id, user, what="pin")
|
||||
db.delete(pin)
|
||||
db.commit()
|
||||
return {"deleted": pin_id}
|
||||
@@ -180,18 +193,11 @@ def reorder_pins(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Rewrite the shelf order from a full list of pin ids.
|
||||
"""Rewrite the shelf order from the list of pin ids the client shows.
|
||||
|
||||
Whole-list rather than a move-one delta: the shelf is short, and a full rewrite cannot leave two
|
||||
tiles claiming the same slot the way an interrupted sequence of single moves can. Ids that are not
|
||||
this user's are ignored rather than 404'd — a stale tab replaying an order that includes a pin
|
||||
deleted meanwhile should reorder what still exists, not fail wholesale."""
|
||||
owned = {p.id for p in pins.list_pins(db, user)}
|
||||
for position, pin_id in enumerate(i for i in body.ids if i in owned):
|
||||
db.execute(
|
||||
update(ComfortPin)
|
||||
.where(ComfortPin.id == pin_id, ComfortPin.user_id == user.id)
|
||||
.values(position=position)
|
||||
)
|
||||
tiles claiming the same slot the way an interrupted sequence of single moves can. The list may be
|
||||
PARTIAL (hidden Plex pins, a stale tab) — `pins.reorder` keeps what it leaves out in place."""
|
||||
pins.reorder(db, user, body.ids)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
|
||||
+208
-99
@@ -9,20 +9,24 @@ serves finished files (range-aware, with the user's custom display name), and sh
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated
|
||||
from app import audit, sysconfig
|
||||
from app.auth import admin_user, require_human
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.downloads import credentials as dlcreds
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import links as linksmod
|
||||
@@ -37,10 +41,13 @@ from app.models import (
|
||||
DownloadShare,
|
||||
MediaAsset,
|
||||
User,
|
||||
display_text,
|
||||
)
|
||||
from app.routes._common import capped_stream_to, get_or_404, not_found, owned_or_404
|
||||
from app.security import hash_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
|
||||
admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"])
|
||||
@@ -62,7 +69,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
as a raw URL for yt-dlp's generic extractor."""
|
||||
raw = (raw or "").strip()
|
||||
if not raw:
|
||||
raise HTTPException(status_code=400, detail="A video link or id is required.")
|
||||
raise refused(400, "link_required")
|
||||
if _VIDEO_ID.match(raw):
|
||||
return "youtube", raw
|
||||
if raw.startswith(("http://", "https://")):
|
||||
@@ -92,7 +99,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
if _VIDEO_ID.match(vid):
|
||||
return "youtube", vid
|
||||
return "url", raw
|
||||
raise HTTPException(status_code=400, detail="That doesn't look like a valid video link.")
|
||||
raise refused(400, "invalid_link")
|
||||
|
||||
|
||||
def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
@@ -106,6 +113,7 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"phase": job.phase,
|
||||
"error": job.error,
|
||||
"error_code": job.error_code,
|
||||
"error_detail": job.error_detail,
|
||||
"queue_pos": job.queue_pos,
|
||||
"created_at": iso(job.created_at),
|
||||
"source_kind": job.source_kind,
|
||||
@@ -164,11 +172,30 @@ def _serialize_job(db: Session, job: DownloadJob) -> dict:
|
||||
return _serialize(job, asset)
|
||||
|
||||
|
||||
def _own_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None or job.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
return job
|
||||
def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
||||
or writes the status those decide on (pause). The job is re-read under a row lock, so two
|
||||
overlapping requests for one job run one after the other and the second decides on the first's
|
||||
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||
just released back into a holder; either can free a file another job still holds.
|
||||
|
||||
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||
(`service.finalize_if_running`), so the two can never deadlock."""
|
||||
while True:
|
||||
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
if job.asset_id == asset_id:
|
||||
return job
|
||||
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||
|
||||
|
||||
# --- profiles ------------------------------------------------------------------------------
|
||||
@@ -212,7 +239,7 @@ def create_profile(
|
||||
name = body.name.strip()
|
||||
spec = body.spec
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="A profile name is required.")
|
||||
raise refused(400, "profile_name_required")
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
|
||||
DownloadProfile.user_id == user.id
|
||||
@@ -240,16 +267,16 @@ def update_profile(
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="A profile name is required.")
|
||||
raise refused(400, "profile_name_required")
|
||||
p.name = name[:80]
|
||||
if "spec" in sent:
|
||||
if body.spec is None:
|
||||
raise HTTPException(status_code=400, detail="spec must be an object")
|
||||
raise refused(400, "spec_not_object")
|
||||
p.spec = body.spec
|
||||
db.commit()
|
||||
return _serialize_profile(p)
|
||||
@@ -261,7 +288,7 @@ def delete_profile(
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
db.delete(p)
|
||||
db.commit()
|
||||
return {"deleted": profile_id}
|
||||
@@ -291,7 +318,7 @@ def set_credential(
|
||||
"""Store (encrypted) this user's cookies.txt for one supported host. Write-only: the value is
|
||||
never echoed back, logged, or audited — only the fact that it changed."""
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise HTTPException(status_code=404, detail="Unknown host")
|
||||
raise not_found("host")
|
||||
try:
|
||||
cleaned = dlcreds.validate_cookies(body.cookies)
|
||||
except dlcreds.InvalidCookies as e:
|
||||
@@ -313,7 +340,7 @@ def delete_credential(
|
||||
host_key: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise HTTPException(status_code=404, detail="Unknown host")
|
||||
raise not_found("host")
|
||||
if dlcreds.delete_credential(db, user.id, host_key):
|
||||
audit.record(
|
||||
db,
|
||||
@@ -334,7 +361,7 @@ def _resolve_spec(
|
||||
if profile_id is not None:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or (not p.is_builtin and p.user_id != user.id):
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
return p.spec, p.id
|
||||
if spec is not None:
|
||||
return spec, None
|
||||
@@ -345,7 +372,7 @@ def _resolve_spec(
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if fallback is None:
|
||||
raise HTTPException(status_code=400, detail="No download profile available.")
|
||||
raise refused(400, "no_profile")
|
||||
return fallback.spec, fallback.id
|
||||
|
||||
|
||||
@@ -376,7 +403,7 @@ def enqueue_download(
|
||||
status_code=422, detail={"code": "url_blocked", "reason": e.reason}
|
||||
) from e
|
||||
spec, profile_id = _resolve_spec(db, user, body.profile_id, body.spec)
|
||||
display_name = (body.display_name or "").strip() or None
|
||||
display_name = display_text(body.display_name)
|
||||
try:
|
||||
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
|
||||
except quota.QuotaExceeded as e:
|
||||
@@ -384,61 +411,119 @@ def enqueue_download(
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
# Read the upload in 1 MiB chunks so a large file never sits fully in memory, and the size cap is
|
||||
# enforced on the ACTUAL bytes read (a client-set Content-Length can't be trusted to gate it).
|
||||
_UPLOAD_CHUNK = 1024 * 1024
|
||||
# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
|
||||
# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
|
||||
# FastAPI *before the first dependency runs*, so `require_human` could not gate it and an anonymous
|
||||
# caller could make the process write the whole per-file allowance to temp, per request (P1; SEC-20
|
||||
# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
|
||||
# declared AFTER the auth one, is the only shape that closes it — the same one
|
||||
# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
|
||||
# every route with a raised ceiling rather than leaving it to a comment like this one.
|
||||
#
|
||||
# Headers are latin-1 on the wire, so both are percent-encoded by the client and unquoted here; a
|
||||
# Hungarian filename would otherwise be unsendable.
|
||||
UPLOAD_FILENAME_HEADER = "x-upload-filename"
|
||||
UPLOAD_DISPLAY_NAME_HEADER = "x-upload-display-name"
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
def upload_download(
|
||||
file: UploadFile = File(...),
|
||||
display_name: str | None = Form(None),
|
||||
@dataclass(frozen=True)
|
||||
class SpooledUpload:
|
||||
"""What the body dependency hands the handler: the bytes already on disk, plus the two names
|
||||
that came alongside them."""
|
||||
|
||||
path: Path
|
||||
size: int
|
||||
user_id: int
|
||||
filename: str
|
||||
display_name: str | None
|
||||
|
||||
|
||||
def _header_text(request: Request, name: str) -> str:
|
||||
"""One percent-encoded header, decoded. A header that is not valid UTF-8 once unquoted is the
|
||||
caller's doing, not a server fault — it degrades to empty rather than 500ing, and the route's
|
||||
own fallback names the file."""
|
||||
raw = request.headers.get(name, "")
|
||||
try:
|
||||
return unquote(raw, errors="strict").strip()
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
# silent-ok: a mis-encoded header is the caller's own malformed request, and this one is
|
||||
# cosmetic — it names the file. Refusing the upload over it would fail a 500 MiB transfer on
|
||||
# a label; logging it would hand any client a free log line per request.
|
||||
return ""
|
||||
|
||||
|
||||
async def spool_upload(
|
||||
request: Request,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Ingest a local file into the caller's library (E7 S5). Any file type is accepted; media
|
||||
(video/audio) gets the full function set, a generic file is download-only. The file is streamed
|
||||
to a temp path under the download volume (enforcing the per-file cap as it goes), then handed to
|
||||
the non-yt-dlp ingest which probes + places it and returns an already-`done` job.
|
||||
) -> AsyncIterator[SpooledUpload]:
|
||||
"""Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
|
||||
|
||||
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||
plus sync DB writes — so it must NOT run on the event loop. `file.file` is the sync file object
|
||||
Starlette spools the multipart body into."""
|
||||
`user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so
|
||||
naming `require_human` here makes the auth check a precondition of reading the body at all — a
|
||||
stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest
|
||||
MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit,
|
||||
empty, quota, probe or move error, a client that vanished mid-upload — it is removed."""
|
||||
cap = sysconfig.get_int(db, "download_upload_max_bytes")
|
||||
user_id = user.id
|
||||
# Hand the pooled connection back BEFORE the transfer starts — see `db.release_read_connection`
|
||||
# for why, and for the precondition it enforces. Everything this dependency still needs is
|
||||
# already an int or a header in hand: touching an ORM attribute below would reload `user` and
|
||||
# check a connection straight back out.
|
||||
release_read_connection(db)
|
||||
staging = Path(settings.download_root) / ".staging" / "uploads"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=str(staging))
|
||||
tmp_path = Path(tmp)
|
||||
try:
|
||||
size = 0
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
while chunk := file.file.read(_UPLOAD_CHUNK):
|
||||
size += len(chunk)
|
||||
if cap and size > cap:
|
||||
raise HTTPException(
|
||||
status_code=422, detail={"code": "upload_too_large", "limit": cap}
|
||||
)
|
||||
out.write(chunk)
|
||||
if size == 0:
|
||||
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
||||
try:
|
||||
job = uploadmod.ingest_upload(
|
||||
db,
|
||||
user.id,
|
||||
filename=file.filename or "upload",
|
||||
src_path=tmp_path,
|
||||
size=size,
|
||||
display_name=(display_name or "").strip() or None,
|
||||
size = await capped_stream_to(
|
||||
request,
|
||||
out,
|
||||
cap,
|
||||
HTTPException(status_code=422, detail={"code": "upload_too_large", "limit": cap}),
|
||||
)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||
return _serialize_job(db, job)
|
||||
yield SpooledUpload(
|
||||
path=tmp_path,
|
||||
size=size,
|
||||
user_id=user_id,
|
||||
filename=_header_text(request, UPLOAD_FILENAME_HEADER) or "upload",
|
||||
display_name=display_text(_header_text(request, UPLOAD_DISPLAY_NAME_HEADER)),
|
||||
)
|
||||
finally:
|
||||
# Ingest MOVES the temp file into place on success (unlink is then a no-op); on any early exit
|
||||
# — cap hit, empty, quota, probe/move error — this removes the abandoned staging file.
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
def upload_download(
|
||||
upload: SpooledUpload = Depends(spool_upload),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Ingest a local file into the caller's library (E7 S5). Any file type is accepted; media
|
||||
(video/audio) gets the full function set, a generic file is download-only. The body IS the file
|
||||
(see the header note above); `spool_upload` has already authenticated the caller and streamed it
|
||||
to a temp path under the download volume, so what is left here is the non-yt-dlp ingest which
|
||||
probes + places it and returns an already-`done` job.
|
||||
|
||||
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||
plus sync DB writes — so it must NOT run on the event loop. The streaming half is async because
|
||||
reading the body is; keeping them in separate callables is what lets each run where it belongs."""
|
||||
if upload.size == 0:
|
||||
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
||||
try:
|
||||
job = uploadmod.ingest_upload(
|
||||
db,
|
||||
upload.user_id,
|
||||
filename=upload.filename,
|
||||
src_path=upload.path,
|
||||
size=upload.size,
|
||||
display_name=upload.display_name,
|
||||
)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
class EnqueueEditIn(BaseModel):
|
||||
source_job_id: int = 0
|
||||
edit_spec: dict
|
||||
@@ -456,9 +541,9 @@ def enqueue_edit(
|
||||
source file is only read, never modified, so editing a shared video is safe."""
|
||||
source_job = _accessible_job(db, user, body.source_job_id)
|
||||
if source_job.status != "done" or source_job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="You can only edit a finished download.")
|
||||
raise refused(409, "edit_unfinished")
|
||||
spec = body.edit_spec
|
||||
display_name = (body.display_name or "").strip() or None
|
||||
display_name = display_text(body.display_name)
|
||||
try:
|
||||
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
|
||||
except quota.QuotaExceeded as e:
|
||||
@@ -494,7 +579,20 @@ def list_downloads(
|
||||
|
||||
@router.get("/usage")
|
||||
def my_usage(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
return quota.usage(db, user.id)
|
||||
"""The caller's download quota, plus the per-file upload cap.
|
||||
|
||||
The cap rides here rather than in `quota.usage` because it is not a quota value — it is a
|
||||
`sysconfig` knob about one route — and this is the route layer, where composing the two is the
|
||||
job. The upload dialog needs it to refuse an over-cap file BEFORE sending it: both server-side
|
||||
caps answer too late to be pleasant (the edge ceiling refuses the request outright, see
|
||||
`bodylimit.ROUTE_MAX_BODY_BYTES`), and making the user push 600 MB up the wire to be told the
|
||||
limit is 500 is a bad answer even when it is a correct one. It is a convenience, not the
|
||||
boundary: the caps stay where they are and still refuse a client that ignores this.
|
||||
"""
|
||||
return {
|
||||
**quota.usage(db, user.id),
|
||||
"upload_max_bytes": sysconfig.get_int(db, "download_upload_max_bytes"),
|
||||
}
|
||||
|
||||
|
||||
# Priority when a source has several jobs (e.g. a done one plus a fresh queued one).
|
||||
@@ -559,6 +657,8 @@ def _clean_url(raw) -> str | None:
|
||||
# authority. This one already has a "not a plausible URL" answer, so the typo belongs in it.
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
# silent-ok: a typo in a link someone pasted. None is this function's "not a plausible URL"
|
||||
# answer, the caller already has one, and any line here would carry the URL itself.
|
||||
except ValueError:
|
||||
return None
|
||||
if not parsed.netloc:
|
||||
@@ -584,12 +684,12 @@ def update_download_meta(
|
||||
) -> dict:
|
||||
"""Update a download's display metadata: title, channel name + link, and extra reference URLs.
|
||||
All are display-only overrides — the on-disk file keeps its system-decided, id-bound name."""
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
sent = body.model_fields_set
|
||||
if "display_name" in sent:
|
||||
job.display_name = ((body.display_name or "").strip())[:255] or None
|
||||
job.display_name = display_text(body.display_name)
|
||||
if "display_uploader" in sent:
|
||||
job.display_uploader = ((body.display_uploader or "").strip())[:255] or None
|
||||
job.display_uploader = display_text(body.display_uploader)
|
||||
if "display_uploader_url" in sent:
|
||||
job.display_uploader_url = _clean_url(body.display_uploader_url)
|
||||
if "extra_links" in sent:
|
||||
@@ -604,7 +704,7 @@ def update_download_meta(
|
||||
def pause_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.pause(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -613,7 +713,7 @@ def pause_download(
|
||||
def resume_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.resume(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -628,18 +728,19 @@ def redownload_download(
|
||||
resolves to the newly-downloaded file — a re-download never breaks a link you already sent out
|
||||
(only a *failed* re-download leaves the job in `error`, which is the intended broken-link case).
|
||||
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the job
|
||||
id + current asset id) rather than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the new
|
||||
asset's own id, and the user's cookie salt when one applies — `credentials.rebuild_salt`) rather
|
||||
than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
under the current format/naming rules — never silently reuses a shared cached file — and (b)
|
||||
never deletes a file another job/user still shares: the old asset is only removed if THIS job
|
||||
was its last holder."""
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
if job.job_kind in ("edit", "upload"):
|
||||
# Neither has a re-fetchable source: an editor clip is derived locally, an upload IS the
|
||||
# source. Re-running yt-dlp against source_kind='upload'/'edit' would just fail.
|
||||
raise HTTPException(status_code=409, detail="This item can't be re-downloaded.")
|
||||
raise refused(409, "not_redownloadable")
|
||||
if job.status in ("queued", "running", "paused"):
|
||||
raise HTTPException(status_code=409, detail="This download is still in progress.")
|
||||
raise refused(409, "still_downloading")
|
||||
service.redownload(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -648,7 +749,7 @@ def redownload_download(
|
||||
def cancel_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.cancel(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -657,7 +758,7 @@ def cancel_download(
|
||||
def delete_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.delete(db, job)
|
||||
return {"deleted": job_id}
|
||||
|
||||
@@ -668,7 +769,7 @@ def delete_download(
|
||||
def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
raise not_found("download")
|
||||
if job.user_id == user.id:
|
||||
return job
|
||||
shared = db.execute(
|
||||
@@ -677,7 +778,7 @@ def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
)
|
||||
).first()
|
||||
if shared is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
@@ -691,10 +792,10 @@ def download_file(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=409, detail="This download isn't ready.")
|
||||
raise refused(409, "download_not_ready")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="File is no longer available.")
|
||||
raise not_found("file")
|
||||
|
||||
filename = storage.download_filename(
|
||||
job.display_name or asset.title or asset.source_ref, asset.container, path
|
||||
@@ -733,7 +834,7 @@ def storyboard_meta(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=409, detail="This download isn't ready.")
|
||||
raise refused(409, "download_not_ready")
|
||||
meta = editmod.ensure_storyboard(asset, settings.download_root)
|
||||
if meta is None:
|
||||
return {"available": False}
|
||||
@@ -754,10 +855,10 @@ def poster_image(job_id: int, user: User = Depends(require_human), db: Session =
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.poster_path:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
|
||||
@@ -768,10 +869,10 @@ def storyboard_image(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.storyboard_path:
|
||||
raise HTTPException(status_code=404, detail="No filmstrip.")
|
||||
raise not_found("filmstrip")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.storyboard_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No filmstrip.")
|
||||
raise not_found("filmstrip")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
|
||||
@@ -789,21 +890,21 @@ def share_download(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="Only a finished download can be shared.")
|
||||
raise refused(409, "share_unfinished")
|
||||
target = body.email.strip().lower()
|
||||
if not target:
|
||||
raise HTTPException(status_code=400, detail="A recipient email is required.")
|
||||
raise refused(400, "recipient_required")
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == target)
|
||||
).scalar_one_or_none()
|
||||
# Uniform "no user" for a missing OR non-messageable (demo/suspended/deactivated) target, so
|
||||
# sharing can't be used to probe which addresses exist or to reach a suspended account.
|
||||
if not is_messageable_user(recipient):
|
||||
raise HTTPException(status_code=404, detail="No user with that email.")
|
||||
raise not_found("recipient")
|
||||
if recipient.id == user.id:
|
||||
raise HTTPException(status_code=400, detail="That's already your download.")
|
||||
raise refused(400, "share_own")
|
||||
exists = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == recipient.id
|
||||
@@ -818,11 +919,18 @@ def share_download(
|
||||
user_id=recipient.id,
|
||||
type="download_shared",
|
||||
title=(job.display_name or "A download") + " was shared with you",
|
||||
data={"kind": "download_shared", "from": user.email, "job_id": job_id},
|
||||
# The client names the notice in the reader's language from `data` (E30); the title is the
|
||||
# English fallback only.
|
||||
data={
|
||||
"kind": "download_shared",
|
||||
"from": user.email,
|
||||
"job_id": job_id,
|
||||
"title": job.display_name,
|
||||
},
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
return {"shared_with": recipient.email}
|
||||
return {"ok": True, "shared_with": recipient.email}
|
||||
|
||||
|
||||
@router.delete("/{job_id}/share/{email}")
|
||||
@@ -832,7 +940,9 @@ def unshare_download(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
_own_job(db, user, job_id) # ownership guard (404s if not the caller's job)
|
||||
owned_or_404(
|
||||
db, DownloadJob, job_id, user, what="download"
|
||||
) # ownership guard (404s if not the caller's job)
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == email.strip().lower())
|
||||
).scalar_one_or_none()
|
||||
@@ -864,7 +974,7 @@ def remove_shared_with_me(
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"removed": job_id}
|
||||
return {"ok": True, "removed": job_id}
|
||||
|
||||
|
||||
# --- recipients (for the internal "share with a user" picker) ------------------------------
|
||||
@@ -880,7 +990,7 @@ def share_recipients(
|
||||
db.execute(
|
||||
select(User)
|
||||
.where(User.id != user.id, *messageable_clauses())
|
||||
.order_by(func.lower(User.email))
|
||||
.order_by(collated(User.email, lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -894,10 +1004,10 @@ def share_recipients(
|
||||
def _own_link(db: Session, user: User, link_id: int) -> DownloadLink:
|
||||
link = db.get(DownloadLink, link_id)
|
||||
if link is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown link")
|
||||
raise not_found("link")
|
||||
job = db.get(DownloadJob, link.job_id)
|
||||
if job is None or job.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown link")
|
||||
raise not_found("link")
|
||||
return link
|
||||
|
||||
|
||||
@@ -913,7 +1023,7 @@ def _link_expiry(days: int | None) -> datetime | None:
|
||||
def list_links(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
_own_job(db, user, job_id)
|
||||
owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
rows = (
|
||||
db.execute(
|
||||
select(DownloadLink)
|
||||
@@ -944,9 +1054,9 @@ def create_link(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="Only a finished download can be shared.")
|
||||
raise refused(409, "share_unfinished")
|
||||
pw = body.password.strip()
|
||||
link = DownloadLink(
|
||||
job_id=job_id,
|
||||
@@ -1091,8 +1201,7 @@ def admin_set_quota(
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if db.get(User, user_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown user")
|
||||
get_or_404(db, User, user_id, what="user")
|
||||
cur = quota.resolve(db, user_id)
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is None:
|
||||
|
||||
+30
-19
@@ -1,7 +1,7 @@
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -10,6 +10,7 @@ from sqlalchemy.orm.exc import StaleDataError
|
||||
from app import quota
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.mailtext import lang_for_user
|
||||
from app.feedquery import (
|
||||
decode_cursor,
|
||||
encode_cursor,
|
||||
@@ -22,15 +23,18 @@ from app.feedquery import (
|
||||
sort_keys,
|
||||
)
|
||||
from app.models import (
|
||||
POSITION_MAX_S,
|
||||
ChannelTag,
|
||||
Tag,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.routes._common import get_or_404, youtube_refusal, not_found
|
||||
from app.sync.videos import parse_iso8601_duration
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["feed"])
|
||||
|
||||
@@ -105,7 +109,7 @@ def get_feed(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys(sort, seed, rank_expr)
|
||||
keys = sort_keys(sort, seed, rank_expr, lang_for_user(user))
|
||||
|
||||
# Expose each key column on the row so we can build the next cursor from the last item.
|
||||
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
||||
@@ -118,7 +122,7 @@ def get_feed(
|
||||
vals, cursor_id = decode_cursor(keys, cursor)
|
||||
except ValueError:
|
||||
# `from None`: the ValueError is an expected bad-input signal, not an error to chain.
|
||||
raise HTTPException(status_code=400, detail="Invalid feed cursor") from None
|
||||
raise refused(400, "invalid_cursor") from None
|
||||
for k, v in zip(keys, vals):
|
||||
k["value"] = v
|
||||
query = query.where(keyset_predicate(keys, cursor_id))
|
||||
@@ -233,9 +237,8 @@ def set_video_state(
|
||||
) -> dict:
|
||||
status = body.status
|
||||
if status not in VALID_STATES:
|
||||
raise HTTPException(status_code=400, detail=f"status must be one of {VALID_STATES}")
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise refused(400, "invalid_status")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
@@ -255,9 +258,11 @@ def set_video_state(
|
||||
db.delete(row)
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: the documented lost race — a concurrent player checkpoint already removed
|
||||
# or moved this row. The state the caller asked for is the state that holds either way.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "status": "new"}
|
||||
return {"ok": True, "video_id": video_id, "status": "new"}
|
||||
|
||||
# watched / hidden: atomic upsert. The player marks "watched" at end-of-playback at
|
||||
# the same instant a progress checkpoint may DELETE the in-progress row; a plain
|
||||
@@ -278,7 +283,7 @@ def set_video_state(
|
||||
)
|
||||
db.execute(stmt)
|
||||
db.commit()
|
||||
return {"video_id": video_id, "status": status}
|
||||
return {"ok": True, "video_id": video_id, "status": status}
|
||||
|
||||
|
||||
@router.delete("/videos/{video_id}/state")
|
||||
@@ -297,17 +302,23 @@ def clear_video_state(
|
||||
db.delete(row)
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
|
||||
|
||||
class VideoProgressIn(BaseModel):
|
||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||
# one into a 500.
|
||||
model_config = ConfigDict(allow_inf_nan=False)
|
||||
|
||||
# Accept a float and truncate (below) — the player may post a raw currentTime (a float), which
|
||||
# the old `int(payload.get(...))` silently truncated. A plain `int` field would 422 that on the
|
||||
# 10s progress hot path, losing the resume position; `float` keeps the old tolerance.
|
||||
position_seconds: float = 0
|
||||
duration_seconds: float = 0
|
||||
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
||||
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
|
||||
|
||||
@router.post("/videos/{video_id}/progress")
|
||||
@@ -326,8 +337,7 @@ def set_video_progress(
|
||||
if position < 0:
|
||||
position = 0
|
||||
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
# Decide whether this position is worth keeping (else clear it).
|
||||
near_end = duration > 0 and position > duration - FINISH_MARGIN_SECONDS
|
||||
@@ -350,9 +360,10 @@ def set_video_progress(
|
||||
# have removed/updated this row already — tolerate the lost race.
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: exactly the race the comment above describes; the response stays true.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "position_seconds": 0}
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": 0}
|
||||
|
||||
if row is None:
|
||||
row = VideoState(user_id=user.id, video_id=video_id)
|
||||
@@ -360,7 +371,7 @@ def set_video_progress(
|
||||
row.position_seconds = position
|
||||
row.progress_updated_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
return {"video_id": video_id, "position_seconds": position}
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": position}
|
||||
|
||||
|
||||
@router.get("/videos/{video_id}")
|
||||
@@ -393,7 +404,7 @@ def get_video_detail(
|
||||
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit, so the
|
||||
# demo account can't reach it — it just sees "unknown" for a video we don't already store.
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise not_found("video")
|
||||
|
||||
try:
|
||||
with (
|
||||
@@ -402,9 +413,9 @@ def get_video_detail(
|
||||
):
|
||||
items = yt.get_videos([video_id])
|
||||
except YouTubeError as exc:
|
||||
raise HTTPException(status_code=422, detail=f"YouTube lookup failed: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
if not items:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise not_found("video")
|
||||
|
||||
snippet = items[0].get("snippet", {})
|
||||
stats = items[0].get("statistics", {})
|
||||
|
||||
+80
-25
@@ -1,5 +1,7 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, RootModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -7,6 +9,7 @@ from app.auth import (
|
||||
count_admins,
|
||||
count_pending_activations,
|
||||
current_user,
|
||||
drop_from_wallet,
|
||||
google_enabled,
|
||||
has_read_scope,
|
||||
has_write_scope,
|
||||
@@ -14,10 +17,16 @@ from app.auth import (
|
||||
optional_current_user,
|
||||
purge_user,
|
||||
)
|
||||
from app import debugsession, diagnostics
|
||||
from app.issues import gitea
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import sysconfig
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/me", tags=["me"])
|
||||
|
||||
@@ -64,28 +73,28 @@ def switch_account(
|
||||
here) are allowed, and access is re-checked in case the invite was revoked since."""
|
||||
target = body.user_id
|
||||
if target not in (request.session.get("account_ids") or []):
|
||||
raise HTTPException(status_code=403, detail="Not an account you've signed into here.")
|
||||
raise refused(403, "not_your_account")
|
||||
# An account `_session_user` would refuse — gone, deactivated, suspended, or its session revoked — is
|
||||
# refused here too, and dropped from the switch list the way `_session_user` drops it. Made the session
|
||||
# default instead, the next request's current_user would refuse it as the DEFAULT and clear the whole
|
||||
# wallet session, logging out every account here. (B-core-9)
|
||||
u = db.get(User, target)
|
||||
if u is None:
|
||||
raise HTTPException(status_code=404, detail="That account no longer exists.")
|
||||
drop_from_wallet(request.session, target)
|
||||
raise not_found("account")
|
||||
if not is_allowed(db, u.email):
|
||||
raise HTTPException(status_code=403, detail="That account no longer has access.")
|
||||
raise refused(403, "account_no_access")
|
||||
if not u.is_active:
|
||||
drop_from_wallet(request.session, target)
|
||||
raise refused(403, "account_no_access")
|
||||
if u.is_suspended:
|
||||
# Don't make a suspended account the active one — the next request's current_user would
|
||||
# see the suspension and clear the WHOLE wallet session, logging out every account here.
|
||||
raise HTTPException(status_code=403, detail="That account is suspended.")
|
||||
# Same reasoning for a REVOKED session (a password reset/change or "log out everywhere" bumped the
|
||||
# account's epoch): making it the session default would trip current_user's epoch check on the next
|
||||
# request and clear the whole wallet. Mirror the suspend guard — drop the stale account from the
|
||||
# switch list and refuse, rather than logging everyone out. (B-core-9)
|
||||
drop_from_wallet(request.session, target)
|
||||
raise refused(403, "account_suspended")
|
||||
# Revoked: a password reset/change or "log out everywhere" bumped the account's epoch.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(target), 0)) != (u.session_epoch or 0):
|
||||
request.session["account_ids"] = [
|
||||
a for a in (request.session.get("account_ids") or []) if a != target
|
||||
]
|
||||
raise HTTPException(
|
||||
status_code=403, detail="That account was signed out elsewhere — sign in again."
|
||||
)
|
||||
drop_from_wallet(request.session, target)
|
||||
raise refused(403, "account_signed_out")
|
||||
request.session["user_id"] = target
|
||||
return {"ok": True, "user_id": target}
|
||||
|
||||
@@ -131,6 +140,9 @@ def get_me(
|
||||
user.google_email if user.google_email and user.google_email != user.email else None
|
||||
),
|
||||
"has_password": user.password_hash is not None,
|
||||
# The account's latest sign-in on any device — shown in Settings → Account, so a sign-in the
|
||||
# user does not recognise is something they can see (null: none since 0.88, or the demo).
|
||||
"last_login_at": iso(user.last_login_at),
|
||||
# Instance-wide: whether Google sign-in / YouTube connect is available at all (the UI hides
|
||||
# YouTube-access affordances and the onboarding nudge when it isn't configured).
|
||||
"google_enabled": google_enabled(db),
|
||||
@@ -144,9 +156,21 @@ def get_me(
|
||||
"pending_activations": pending_activations,
|
||||
"email_configured": email_configured,
|
||||
"preferences": user.preferences or {},
|
||||
# The consented diagnostics level in force and when it lapses (E22 S5). The client applies
|
||||
# it to its own log and streams that log while it is raised.
|
||||
"diagnostics": diagnostics.state(db, user),
|
||||
# The live debug session, if one is recording (E16) — a reload resumes it.
|
||||
"debug_session": _live_session(db, user),
|
||||
# Where a bug report goes (E16 S4): the tracker's repository, or None — download only.
|
||||
"issues": gitea.target(db),
|
||||
}
|
||||
|
||||
|
||||
def _live_session(db: Session, user: User) -> dict | None:
|
||||
session = debugsession.open_session(db, user)
|
||||
return None if session is None else debugsession.view(db, session)
|
||||
|
||||
|
||||
@router.delete("/account")
|
||||
def delete_my_account(
|
||||
request: Request,
|
||||
@@ -159,13 +183,15 @@ def delete_my_account(
|
||||
which all cascade on the users row. Quota-audit events are kept but anonymised (FK SET NULL).
|
||||
The shared demo account can't be deleted, and the last remaining admin can't delete itself
|
||||
(that would lock everyone out of the admin surfaces)."""
|
||||
# Structured, like the other three demo refusals: `req` leaves 403 to the caller and
|
||||
# `notifyActionError` discards a plain-string detail in favour of the caller's translated
|
||||
# fallback, so a hand-written English sentence here reaches a Hungarian UI as a generic
|
||||
# "couldn't do that" — or, on this affordance, as nothing at all. `reason` distinguishes it
|
||||
# from the other sites' bare `{code: "demo"}` so the UI can say what specifically is refused.
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="The shared demo account can't be deleted.")
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "delete_account"})
|
||||
if user.role == "admin" and count_admins(db) <= 1:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="You're the only admin — promote another admin before deleting your account.",
|
||||
)
|
||||
raise refused(400, "only_admin_delete")
|
||||
user_id = user.id
|
||||
# Full erasure (cascades) + access-request cleanup + Google-grant revocation; shared with the
|
||||
# admin delete path. Capture the id first — `user` is detached once purge_user deletes the row.
|
||||
@@ -180,16 +206,45 @@ def delete_my_account(
|
||||
return {"deleted": True}
|
||||
|
||||
|
||||
class PreferencesIn(RootModel[dict[str, Any]]):
|
||||
"""Free-form UI preferences — a JSON object of any keys (theme, filters, …), merged into the
|
||||
stored ones. A model rather than a bare `dict` parameter so a non-object body is a 422 at the
|
||||
boundary like every other request body."""
|
||||
|
||||
|
||||
@router.put("/preferences")
|
||||
def update_preferences(
|
||||
preferences: dict,
|
||||
preferences: PreferencesIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# Merge so partial updates don't wipe other keys.
|
||||
merged = dict(user.preferences or {})
|
||||
merged.update(preferences)
|
||||
merged.update(preferences.root)
|
||||
user.preferences = merged
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return {"preferences": merged}
|
||||
return {"ok": True, "preferences": merged}
|
||||
|
||||
|
||||
class DiagnosticsIn(BaseModel):
|
||||
level: diagnostics.DiagnosticsLevel
|
||||
|
||||
|
||||
@router.put("/diagnostics")
|
||||
def update_diagnostics(
|
||||
body: DiagnosticsIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Raise the diagnostics level (the user consented in Settings → Diagnosztika) or return to the
|
||||
base. The expiry is the server's: a raised level always lapses after
|
||||
`diagnostics.ELEVATION_HOURS`, whatever the client asks. The shared demo account cannot raise
|
||||
it — its log would interleave every visitor's session under one account."""
|
||||
if user.is_demo and body.level != diagnostics.BASE:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "diagnostics"})
|
||||
diagnostics.set_level(user, body.level)
|
||||
db.commit()
|
||||
# The rest of THIS request, and the process's elevation registry, follow the new level now.
|
||||
logs.set_user(user.id, diagnostics.effective_level(user), user.diagnostics_until)
|
||||
return {"ok": True, "diagnostics": diagnostics.state(db, user)}
|
||||
|
||||
@@ -16,11 +16,13 @@ routing and unread counts.
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, WebSocket, WebSocketDisconnect
|
||||
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, case, delete, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated
|
||||
from app.auth import require_human, resolved_user_id
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import Message, MessageKey, User
|
||||
@@ -29,6 +31,8 @@ from app.realtime import manager
|
||||
from app.security import verify_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/messages", tags=["messages"])
|
||||
|
||||
@@ -46,7 +50,7 @@ WELCOME = {
|
||||
"This is your private message inbox. Conversations with other members are end-to-end "
|
||||
"encrypted — only you and the person you're chatting with can read them, not even an "
|
||||
"admin. To start messaging someone, set up secure messaging with a passphrase first.\n\n"
|
||||
"Enjoy your feed!"
|
||||
"Enjoy your Feed!"
|
||||
),
|
||||
"hu": (
|
||||
"Üdv a Siftlode-on! 👋\n\n"
|
||||
@@ -152,7 +156,7 @@ def setup_keys(
|
||||
"""Store this user's public key + passphrase-wrapped private key. Set-once: changing the
|
||||
key would orphan every existing conversation key, so re-setup is rejected here."""
|
||||
if db.get(MessageKey, user.id) is not None:
|
||||
raise HTTPException(status_code=409, detail="Secure messaging is already set up.")
|
||||
raise refused(409, "messaging_set_up")
|
||||
db.add(
|
||||
MessageKey(
|
||||
user_id=user.id,
|
||||
@@ -164,7 +168,7 @@ def setup_keys(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"configured": True}
|
||||
return {"ok": True, "configured": True}
|
||||
|
||||
|
||||
@router.post("/keys/reset")
|
||||
@@ -185,11 +189,11 @@ def reset_keys(
|
||||
less (Google-only) account has nothing to re-verify, so it relies on its live session plus the
|
||||
client-side danger confirmation."""
|
||||
if not _reset_key_limiter.allow(str(user.id)):
|
||||
raise HTTPException(status_code=429, detail="Too many attempts. Try again shortly.")
|
||||
raise refused(429, "too_many_attempts")
|
||||
if user.password_hash and not verify_password(
|
||||
payload.current_password or "", user.password_hash
|
||||
):
|
||||
raise HTTPException(status_code=403, detail="Current password is incorrect.")
|
||||
raise refused(403, "wrong_password")
|
||||
# Delete the now-dead ciphertext first (both directions), then the wrapped keypair itself so a
|
||||
# fresh setup is allowed. Order doesn't matter (no FK between them), but do it in one commit.
|
||||
db.execute(
|
||||
@@ -202,7 +206,7 @@ def reset_keys(
|
||||
if key is not None:
|
||||
db.delete(key)
|
||||
db.commit()
|
||||
return {"configured": False}
|
||||
return {"ok": True, "configured": False}
|
||||
|
||||
|
||||
@router.get("/keys/{user_id}")
|
||||
@@ -223,10 +227,10 @@ def public_key(
|
||||
p = db.get(User, user_id)
|
||||
has_history = db.scalar(select(Message.id).where(cond).limit(1))
|
||||
if p is None or (not is_messageable_user(p) and not has_history):
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
raise not_found("user")
|
||||
k = db.get(MessageKey, user_id)
|
||||
if k is None:
|
||||
raise HTTPException(status_code=404, detail="That member hasn't set up messaging yet.")
|
||||
raise refused(404, "no_messaging")
|
||||
return {"user_id": user_id, "public_key": k.public_key}
|
||||
|
||||
|
||||
@@ -244,7 +248,7 @@ def list_recipients(
|
||||
select(User)
|
||||
.join(MessageKey, MessageKey.user_id == User.id)
|
||||
.where(User.id != user.id, *messageable_clauses())
|
||||
.order_by(func.lower(func.coalesce(User.display_name, User.email)))
|
||||
.order_by(collated(func.coalesce(User.display_name, User.email), lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -344,7 +348,7 @@ def get_thread(
|
||||
# still opens). Otherwise the same 404 as a non-existent id → no enumeration oracle.
|
||||
has_history = db.scalar(select(Message.id).where(cond).limit(1))
|
||||
if p is None or (not is_messageable_user(p) and not has_history):
|
||||
raise HTTPException(status_code=404, detail="User not found")
|
||||
raise not_found("user")
|
||||
partner = _serialize_user(p)
|
||||
msgs = (
|
||||
db.execute(select(Message).where(cond).order_by(Message.created_at.asc())).scalars().all()
|
||||
@@ -374,20 +378,20 @@ def send_message(
|
||||
"""Send an end-to-end-encrypted message. The server stores only the ciphertext the client
|
||||
produced, then pushes it live to the recipient (and the sender's other devices)."""
|
||||
if not _send_limiter.allow(f"msg:{user.id}"):
|
||||
raise HTTPException(status_code=429, detail="Too many messages — slow down a moment.")
|
||||
raise refused(429, "message_rate")
|
||||
ct = (payload.ciphertext or "").strip()
|
||||
iv = (payload.iv or "").strip()
|
||||
if not ct or not iv:
|
||||
raise HTTPException(status_code=400, detail="Message is empty.")
|
||||
raise refused(400, "message_empty")
|
||||
if len(ct) > MAX_CIPHERTEXT:
|
||||
raise HTTPException(status_code=400, detail="Message is too long.")
|
||||
raise refused(400, "message_too_long")
|
||||
if payload.recipient_id == user.id:
|
||||
raise HTTPException(status_code=400, detail="You can't message yourself.")
|
||||
raise refused(400, "message_self")
|
||||
recipient = db.get(User, payload.recipient_id)
|
||||
if not is_messageable_user(recipient):
|
||||
raise HTTPException(status_code=404, detail="Recipient not available.")
|
||||
raise not_found("recipient")
|
||||
if db.get(MessageKey, recipient.id) is None:
|
||||
raise HTTPException(status_code=404, detail="That member hasn't set up messaging yet.")
|
||||
raise refused(404, "no_messaging")
|
||||
m = Message(kind="user", sender_id=user.id, recipient_id=recipient.id, ciphertext=ct, iv=iv)
|
||||
db.add(m)
|
||||
db.commit()
|
||||
@@ -423,9 +427,10 @@ async def messages_ws(ws: WebSocket) -> None:
|
||||
"""Live push channel: authenticated by the session cookie, registers the connection so
|
||||
sent messages reach this user's open tabs instantly. We don't consume client→server frames
|
||||
(sending goes through POST /api/messages); the receive loop only detects disconnect."""
|
||||
# Which signed-in account this socket acts as — the SHARED per-tab resolution (wallet-gated
|
||||
# ?account= override, session default otherwise). A browser WebSocket can't send the
|
||||
# X-Siftlode-Account header, so resolved_user_id falls through to the ?account= query param.
|
||||
# Which signed-in account this socket acts as — the SHARED per-tab resolution (a wallet-gated
|
||||
# ?account=, the session default when none is named, nobody for an account no longer signed in
|
||||
# here). A browser WebSocket can't send the X-Siftlode-Account header, so resolved_user_id falls
|
||||
# through to the ?account= query param.
|
||||
uid, _ = resolved_user_id(ws)
|
||||
if not uid:
|
||||
await ws.close(code=1008)
|
||||
@@ -448,6 +453,8 @@ async def messages_ws(ws: WebSocket) -> None:
|
||||
try:
|
||||
while True:
|
||||
await ws.receive_text()
|
||||
# silent-ok: the ordinary way a chat socket ends — a closed tab. The `finally` below is the
|
||||
# whole handling, and nothing failed.
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
finally:
|
||||
|
||||
@@ -5,7 +5,7 @@ All endpoints use `current_user` (not `require_human`): notifications and their
|
||||
state are personal UI state with no quota or YouTube involvement, so the shared demo account
|
||||
manages its own inbox normally."""
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -14,6 +14,7 @@ from app.db import get_db
|
||||
from app.models import Notification, User
|
||||
from app.notifications import trim_read
|
||||
from app.utils import iso
|
||||
from app.routes._common import owned_or_404
|
||||
|
||||
router = APIRouter(prefix="/api/me/notifications", tags=["notifications"])
|
||||
|
||||
@@ -72,20 +73,13 @@ def unread_count(user: User = Depends(current_user), db: Session = Depends(get_d
|
||||
return {"count": count or 0}
|
||||
|
||||
|
||||
def _owned(db: Session, notification_id: int, user: User) -> Notification:
|
||||
n = db.get(Notification, notification_id)
|
||||
if n is None or n.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Notification not found")
|
||||
return n
|
||||
|
||||
|
||||
@router.post("/{notification_id}/read")
|
||||
def mark_read(
|
||||
notification_id: int,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
n = _owned(db, notification_id, user)
|
||||
n = owned_or_404(db, Notification, notification_id, user, what="notification")
|
||||
n.read = True
|
||||
db.commit()
|
||||
trim_read(db, user.id)
|
||||
@@ -110,7 +104,7 @@ def dismiss(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
n = _owned(db, notification_id, user)
|
||||
n = owned_or_404(db, Notification, notification_id, user, what="notification")
|
||||
# Dismiss implies read (it no longer counts toward the unread badge).
|
||||
n.dismissed = True
|
||||
n.read = True
|
||||
|
||||
+178
-107
@@ -3,10 +3,8 @@ CRUD (create / rename / delete, add / remove / reorder items) plus YouTube two-w
|
||||
mirror the user's own YouTube playlists in (`/sync-youtube`) and push/revert local edits back
|
||||
out (`/{id}/push-plan`, `/{id}/revert-youtube`), behind the optional write scope."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import func, select, and_
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
@@ -16,13 +14,18 @@ from app.db import get_db
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
|
||||
from app.sync.playlists import (
|
||||
PlaylistGone,
|
||||
lock_playlists,
|
||||
merge_playlist,
|
||||
plan_merge,
|
||||
recompute_dirty,
|
||||
pull_user_playlists,
|
||||
repull_playlist,
|
||||
sync_user_playlists,
|
||||
sync_claim,
|
||||
user_sync_claim,
|
||||
)
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
|
||||
router = APIRouter(prefix="/api/playlists", tags=["playlists"])
|
||||
@@ -47,11 +50,10 @@ def _check_pull_cooldown(user: User) -> None:
|
||||
raise HTTPException(status_code=429, detail={"code": "sync_cooldown"})
|
||||
|
||||
|
||||
def _own_playlist(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
pl = db.get(Playlist, playlist_id)
|
||||
if pl is None or pl.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown playlist")
|
||||
return pl
|
||||
def _refused(status: int, reason: str) -> HTTPException:
|
||||
"""A playlist action the server refuses, as a reason the UI translates
|
||||
(`playlists.errors.<reason>`) — a bare English detail reached the Hungarian UI verbatim."""
|
||||
return HTTPException(status_code=status, detail={"code": "playlist", "reason": reason})
|
||||
|
||||
|
||||
def _touch(pl: Playlist) -> None:
|
||||
@@ -60,8 +62,11 @@ def _touch(pl: Playlist) -> None:
|
||||
`UpdatedAtMixin`'s `onupdate` fires on an UPDATE of the PLAYLIST row, and add / remove /
|
||||
reorder / move write the item rows instead — so without this the rail's "recently changed"
|
||||
sort would only ever see renames. The assignment is what makes the column mean "last time
|
||||
this list changed", which is what the rail promises the reader."""
|
||||
pl.updated_at = datetime.now(timezone.utc)
|
||||
this list changed", which is what the rail promises the reader.
|
||||
|
||||
The DATABASE's clock, like the mixin's own `onupdate`: one sort key written from two clocks
|
||||
(this process's and Postgres's) could order a rename and an item edit the wrong way round."""
|
||||
pl.updated_at = func.now()
|
||||
|
||||
|
||||
def _next_playlist_position(db: Session, user_id: int) -> int:
|
||||
@@ -77,7 +82,10 @@ def _next_playlist_position(db: Session, user_id: int) -> int:
|
||||
|
||||
|
||||
def _next_item_position(db: Session, playlist_id: int) -> int:
|
||||
"""Append position for a new item at the end of a playlist."""
|
||||
"""Append position for a new item at the end of a playlist. Locks the playlist row first: two
|
||||
adds (two tabs, the rail and the page) otherwise both read the same max and land on one
|
||||
position, and the list's order between them is then arbitrary."""
|
||||
lock_playlists(db, playlist_id)
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.max(PlaylistItem.position), -1)).where(
|
||||
@@ -90,7 +98,10 @@ def _next_item_position(db: Session, playlist_id: int) -> int:
|
||||
|
||||
def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Append a video to a playlist if not already present (idempotent). Returns whether it
|
||||
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
||||
was added. `mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later).
|
||||
Locked BEFORE the presence check: two adds of one video both saw "absent" and the second one's
|
||||
insert hit uq_playlist_video as a 500."""
|
||||
lock_playlists(db, pl.id)
|
||||
exists = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||
@@ -104,13 +115,14 @@ def _add_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) ->
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
return True
|
||||
|
||||
|
||||
def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool) -> bool:
|
||||
"""Remove a video from a playlist if present (idempotent). Returns whether it was removed.
|
||||
`mark_dirty` recomputes the YouTube-sync dirty flag (skip for Watch later)."""
|
||||
lock_playlists(db, pl.id)
|
||||
item = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == pl.id, PlaylistItem.video_id == video_id
|
||||
@@ -122,7 +134,7 @@ def _remove_item(db: Session, pl: Playlist, video_id: str, *, mark_dirty: bool)
|
||||
if mark_dirty:
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
return True
|
||||
|
||||
|
||||
@@ -256,7 +268,7 @@ def create_playlist(
|
||||
) -> dict:
|
||||
name = body.name.strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name is required")
|
||||
raise _refused(400, "name_required")
|
||||
pl = Playlist(user_id=user.id, name=name, position=_next_playlist_position(db, user.id))
|
||||
db.add(pl)
|
||||
db.commit()
|
||||
@@ -276,7 +288,7 @@ def _watch_later(db: Session, user: User) -> Playlist:
|
||||
position=_next_playlist_position(db, user.id),
|
||||
)
|
||||
db.add(pl)
|
||||
db.commit()
|
||||
db.flush() # the id, for the item the caller adds next; the handler commits
|
||||
return pl
|
||||
|
||||
|
||||
@@ -288,11 +300,11 @@ def add_watch_later(
|
||||
) -> dict:
|
||||
"""Bookmark shortcut: add a video to the built-in Watch later playlist."""
|
||||
video_id = body.video_id
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
pl = _watch_later(db, user)
|
||||
_add_item(db, pl, video_id, mark_dirty=False)
|
||||
return {"saved": True, "playlist_id": pl.id}
|
||||
db.commit()
|
||||
return {"ok": True, "saved": True, "playlist_id": pl.id}
|
||||
|
||||
|
||||
@router.delete("/watch-later/{video_id}")
|
||||
@@ -306,7 +318,8 @@ def remove_watch_later(
|
||||
)
|
||||
if pl is not None:
|
||||
_remove_item(db, pl, video_id, mark_dirty=False)
|
||||
return {"saved": False}
|
||||
db.commit()
|
||||
return {"ok": True, "saved": False}
|
||||
|
||||
|
||||
@router.post("/sync-youtube")
|
||||
@@ -317,13 +330,19 @@ def sync_youtube(
|
||||
"""Pull the user's YouTube playlists into local mirrors now (read-only direction).
|
||||
Also runs automatically on the scheduler; this is the manual 'sync now'."""
|
||||
if not has_read_scope(user):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Connect YouTube (read access) to sync your playlists.",
|
||||
)
|
||||
_check_pull_cooldown(user)
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return sync_user_playlists(db, user)
|
||||
raise _refused(403, "youtube_read_scope")
|
||||
with user_sync_claim(db, user.id) as free:
|
||||
if not free:
|
||||
# Another run (the scheduler, another tab) is pulling right now: "0 synced" would lie.
|
||||
raise _refused(409, "sync_all_busy")
|
||||
# After the claim, like the re-pull's: a refused attempt must not spend one of the pulls.
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return {"ok": True, **pull_user_playlists(db, user)}
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
@router.post("/{playlist_id}/revert-youtube")
|
||||
@@ -335,30 +354,33 @@ def revert_youtube(
|
||||
"""Reset one YouTube-linked playlist to its current YouTube state, discarding local
|
||||
edits (and clearing the dirty flag). The user-facing 'undo all my changes' for a list
|
||||
once the in-session undo history is gone."""
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if not pl.yt_playlist_id:
|
||||
raise HTTPException(status_code=400, detail="This playlist isn't linked to YouTube.")
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_read_scope(user):
|
||||
raise HTTPException(
|
||||
status_code=403, detail="Connect YouTube (read access) to sync your playlists."
|
||||
)
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return repull_playlist(db, user, pl)
|
||||
except YouTubeError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=422, detail="Couldn't refresh from YouTube.")
|
||||
raise _refused(403, "youtube_read_scope")
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
raise _refused(409, "sync_busy")
|
||||
# The cooldown is spent only by a pull that runs: checked after the claim, so clicking
|
||||
# Revert while a push is in flight does not use up the three pulls per ten minutes.
|
||||
_check_pull_cooldown(user)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||
return {"ok": True, **repull_playlist(db, user, pl)}
|
||||
except PlaylistGone:
|
||||
raise not_found("playlist")
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||
"""A local playlist the user may push to YouTube (create or update). The built-in
|
||||
Watch later and read-only YouTube mirrors are never pushable."""
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if pl.kind == "watch_later":
|
||||
raise HTTPException(
|
||||
status_code=400, detail="The Watch later list can't be sent to YouTube."
|
||||
)
|
||||
raise _refused(400, "watch_later_not_pushable")
|
||||
return pl
|
||||
|
||||
|
||||
@@ -375,8 +397,8 @@ def push_plan(
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
plan = plan_merge(db, user, pl)
|
||||
except YouTubeError:
|
||||
raise HTTPException(status_code=422, detail="Couldn't read the playlist on YouTube.")
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc)
|
||||
plan["remaining_today"] = quota.remaining_today(db)
|
||||
plan["affordable"] = plan["units_estimate"] <= plan["remaining_today"]
|
||||
return plan
|
||||
@@ -392,28 +414,32 @@ def push(
|
||||
neither side loses an item). Refuses if the estimated quota exceeds what's left today, so a
|
||||
big sync can't strand half-applied. Write scope required (require_write_scope)."""
|
||||
pl = _pushable(db, user, playlist_id)
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
||||
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||
live = None
|
||||
if pl.yt_playlist_id:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
live = (
|
||||
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||
)
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="Not enough YouTube quota left today for this sync.",
|
||||
)
|
||||
return merge_playlist(db, user, pl, live=live)
|
||||
except YouTubeError:
|
||||
db.rollback()
|
||||
raise HTTPException(status_code=422, detail="YouTube playlist sync failed.")
|
||||
# One sync of a list at a time, across tabs, devices and the scheduler (`sync_claim`): the
|
||||
# client's own in-flight flag only covers the menus of one page.
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
raise _refused(409, "sync_busy")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||
# Read the live playlist ONCE (linked playlists only) and share it with both the
|
||||
# plan and the merge, halving read-quota and closing the second read→write TOCTOU
|
||||
# window. A fresh export (no yt_playlist_id) reads nothing.
|
||||
live = None
|
||||
if pl.yt_playlist_id:
|
||||
with YouTubeClient(db, user) as yt:
|
||||
live = (
|
||||
list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id)),
|
||||
yt.get_playlist_snippet(pl.yt_playlist_id),
|
||||
)
|
||||
plan = plan_merge(db, user, pl, live=live)
|
||||
if plan["units_estimate"] > quota.remaining_today(db):
|
||||
raise _refused(429, "quota_exceeded")
|
||||
return {"ok": True, **merge_playlist(db, user, pl, live=live)}
|
||||
except PlaylistGone:
|
||||
raise not_found("playlist")
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc)
|
||||
|
||||
|
||||
@router.get("/{playlist_id}")
|
||||
@@ -422,7 +448,7 @@ def get_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
state = aliased(VideoState)
|
||||
rows = db.execute(
|
||||
select(*feed_columns(user, state))
|
||||
@@ -430,7 +456,8 @@ def get_playlist(
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(state, and_(state.video_id == Video.id, state.user_id == user.id))
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
.order_by(PlaylistItem.position)
|
||||
# `id` breaks a position tie the same way on every read (the renumbering add does too).
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).all()
|
||||
return {
|
||||
"id": pl.id,
|
||||
@@ -454,11 +481,12 @@ def rename_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
# Locked: the new name moves `dirty`, which the scheduled mirror refresh reads under this lock.
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||
if "name" in body.model_fields_set:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="name cannot be empty")
|
||||
raise _refused(400, "name_required")
|
||||
if name != pl.name:
|
||||
pl.name = name
|
||||
recompute_dirty(db, pl)
|
||||
@@ -491,38 +519,70 @@ def delete_playlist(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
if pl.kind == "watch_later":
|
||||
raise HTTPException(status_code=400, detail="The Watch later list can't be deleted.")
|
||||
if on_youtube:
|
||||
if not pl.yt_playlist_id:
|
||||
raise HTTPException(status_code=400, detail="This playlist isn't on YouTube.")
|
||||
if not has_write_scope(user):
|
||||
raise HTTPException(status_code=403, detail="Enable YouTube editing in Settings first.")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt.delete_playlist(pl.yt_playlist_id)
|
||||
except YouTubeError:
|
||||
raise HTTPException(status_code=422, detail="YouTube delete failed.")
|
||||
db.delete(pl) # items cascade
|
||||
db.commit()
|
||||
raise _refused(400, "watch_later_undeletable")
|
||||
# Not in the middle of a sync of it (`sync_claim`): a push that read the list before the delete
|
||||
# and wrote YouTube after it planned from no items at all, so it emptied the YouTube playlist a
|
||||
# local-only delete means to keep. The push checks for a vanished list too (`PlaylistGone`);
|
||||
# this refuses the delete while a sync runs rather than race it.
|
||||
with sync_claim(db, pl.id) as free:
|
||||
if not free:
|
||||
raise _refused(409, "sync_busy")
|
||||
if on_youtube:
|
||||
if not pl.yt_playlist_id:
|
||||
raise _refused(400, "not_on_youtube")
|
||||
if not has_write_scope(user):
|
||||
raise _refused(403, "youtube_editing_off")
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_DELETE):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
yt.delete_playlist(pl.yt_playlist_id)
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc)
|
||||
# The playlist first, as every item write: the cascade deletes the items before the list row.
|
||||
lock_playlists(db, pl.id)
|
||||
db.delete(pl) # items cascade
|
||||
db.commit()
|
||||
return {"deleted": playlist_id}
|
||||
|
||||
|
||||
class AddItemIn(VideoRefIn):
|
||||
# Where the video lands among the list's items, in their stored order (0 = first). Omitted, it
|
||||
# is appended. The list page's "Undo" of a removal puts the row back where it was with this —
|
||||
# ONE request, because append-then-reorder leaves it at the end whenever the second call fails.
|
||||
position: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
@router.post("/{playlist_id}/items")
|
||||
def add_item(
|
||||
playlist_id: int,
|
||||
body: VideoRefIn,
|
||||
body: AddItemIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
video_id = body.video_id
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
_add_item(db, pl, video_id, mark_dirty=True)
|
||||
return {"playlist_id": pl.id, "video_id": video_id}
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
added = _add_item(db, pl, video_id, mark_dirty=False)
|
||||
if added and body.position is not None:
|
||||
# Renumber the whole list with the new row at `position` (clamped to the end): positions
|
||||
# can have gaps after removals, so "index N" is only meaningful over the ordered rows.
|
||||
rows = list(
|
||||
db.execute(
|
||||
select(PlaylistItem)
|
||||
.where(PlaylistItem.playlist_id == pl.id)
|
||||
.order_by(PlaylistItem.position, PlaylistItem.id)
|
||||
).scalars()
|
||||
)
|
||||
new = next(r for r in rows if r.video_id == video_id)
|
||||
rows.remove(new)
|
||||
rows.insert(min(body.position, len(rows)), new)
|
||||
for pos, row in enumerate(rows):
|
||||
row.position = pos
|
||||
recompute_dirty(db, pl)
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
|
||||
@router.delete("/{playlist_id}/items/{video_id}")
|
||||
@@ -532,9 +592,10 @@ def remove_item(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
_remove_item(db, pl, video_id, mark_dirty=True)
|
||||
return {"playlist_id": pl.id, "video_id": video_id}
|
||||
db.commit()
|
||||
return {"ok": True, "playlist_id": pl.id, "video_id": video_id}
|
||||
|
||||
|
||||
class MoveItemIn(BaseModel):
|
||||
@@ -556,12 +617,16 @@ def move_item(
|
||||
exactly the kind of half-applied state this codebase treats as a bug rather than an edge
|
||||
case. Idempotent: if the video is already in the target, the source row is simply dropped
|
||||
(moving into a list that already has it is a removal, not a duplicate)."""
|
||||
src = _own_playlist(db, user, playlist_id)
|
||||
dst = _own_playlist(db, user, body.target_playlist_id)
|
||||
src = owned_or_404(db, Playlist, playlist_id, user, what="playlist")
|
||||
# Its own reason: "this playlist doesn't exist any more" about a TARGET that is gone was read as
|
||||
# the list on screen — which is still there.
|
||||
dst = owned_or_404(db, Playlist, body.target_playlist_id, user, what="move_target")
|
||||
if src.id == dst.id:
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "playlist_move", "reason": "same_playlist"}
|
||||
)
|
||||
# Both lists, in id order: a move A→B beside a move B→A would otherwise lock crosswise.
|
||||
lock_playlists(db, src.id, dst.id)
|
||||
item = db.scalar(
|
||||
select(PlaylistItem).where(
|
||||
PlaylistItem.playlist_id == src.id, PlaylistItem.video_id == video_id
|
||||
@@ -577,19 +642,24 @@ def move_item(
|
||||
PlaylistItem.playlist_id == dst.id, PlaylistItem.video_id == video_id
|
||||
)
|
||||
)
|
||||
if already is not None:
|
||||
db.delete(item) # the uq_playlist_video constraint would reject a second row anyway
|
||||
else:
|
||||
dst_changed = False
|
||||
if already is None:
|
||||
# Re-parent the existing row rather than delete+insert: `added_at` ("when I saved this")
|
||||
# survives the move, and there is no window where the video belongs to neither list.
|
||||
# survives the move, and there is no window where the video belongs to neither list. The
|
||||
# check above holds: both lists are locked, and an insert into the target (another tab's
|
||||
# add, a YouTube merge) waits for the lock — its foreign key needs the target row.
|
||||
item.playlist_id = dst.id
|
||||
item.position = _next_item_position(db, dst.id)
|
||||
# Both sides changed, so both get the dirty recompute and the timestamp — the asymmetric
|
||||
# version of this (touching only the source) is the recurring smell CLAUDE.md warns about.
|
||||
dst_changed = True
|
||||
if not dst_changed:
|
||||
db.delete(item)
|
||||
# The source always changed; the target only when the row landed in it. `_touch` on a target
|
||||
# that already had the video promoted an untouched list in the rail's "recently changed" sort.
|
||||
recompute_dirty(db, src)
|
||||
recompute_dirty(db, dst)
|
||||
_touch(src)
|
||||
_touch(dst)
|
||||
if dst_changed:
|
||||
recompute_dirty(db, dst)
|
||||
_touch(dst)
|
||||
db.commit()
|
||||
return {"ok": True, "from_playlist_id": src.id, "to_playlist_id": dst.id, "video_id": video_id}
|
||||
|
||||
@@ -605,7 +675,8 @@ def reorder_items(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pl = _own_playlist(db, user, playlist_id)
|
||||
# Locked: a concurrent add between reading the items and writing the order would collide.
|
||||
pl = owned_or_404(db, Playlist, playlist_id, user, what="playlist", lock=True)
|
||||
order = body.video_ids
|
||||
items = {
|
||||
it.video_id: it
|
||||
@@ -631,4 +702,4 @@ def reorder_items(
|
||||
recompute_dirty(db, pl)
|
||||
_touch(pl)
|
||||
db.commit()
|
||||
return {"playlist_id": pl.id, "count": pos}
|
||||
return {"ok": True, "playlist_id": pl.id, "count": pos}
|
||||
|
||||
@@ -1,23 +1,18 @@
|
||||
"""Shared helpers, serializers and constants for the plex route package."""
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import PlexItem
|
||||
from app.plex.cards import norm_lang as _norm_lang # noqa: F401 — the package's historic name
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.routes._common import not_found
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
||||
it = by_rating_key(db, PlexItem, rating_key)
|
||||
if it is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex item")
|
||||
raise not_found("plex_item")
|
||||
return it
|
||||
|
||||
|
||||
def _norm_lang(lang: str | None) -> str:
|
||||
"""The overlay has an EN base + an optional HU twin; anything other than `hu` serves EN."""
|
||||
return "hu" if lang == "hu" else "en"
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -20,6 +20,9 @@ from app.plex import sync as plex_sync
|
||||
from app.plex import watch_sync as plex_watch
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
@@ -35,10 +38,15 @@ def test_connection(_: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
info = plex.server_info()
|
||||
sections = plex.sections()
|
||||
media_check = _media_check(db, plex, sections)
|
||||
except PlexNotConfigured as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
except PlexNotConfigured:
|
||||
raise refused(400, "plex_not_configured") from None
|
||||
except PlexError as e:
|
||||
raise HTTPException(status_code=422, detail=f"Plex connection failed: {e}")
|
||||
# What the upstream said is for the log, not the toast: it can quote the server URL back.
|
||||
log.warning(
|
||||
"the Plex connection test failed",
|
||||
extra=logs.event("plex.admin.connection_failed", reason=type(e).__name__, error=str(e)),
|
||||
)
|
||||
raise refused(422, "plex_unreachable") from None
|
||||
return {
|
||||
"ok": True,
|
||||
"server_name": info.get("friendlyName") or info.get("title1") or "Plex",
|
||||
@@ -95,7 +103,7 @@ def _media_check(db: Session, plex: PlexClient, sections: list[dict]) -> dict:
|
||||
def trigger_sync(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Admin: mirror the enabled Plex sections into the local catalog. Synchronous (can take a
|
||||
while on a large library); a background/scheduled sync also runs periodically."""
|
||||
return plex_sync.sync(db)
|
||||
return {"ok": True, **plex_sync.sync(db)}
|
||||
|
||||
|
||||
def _link_dict(link: PlexLink | None) -> dict:
|
||||
@@ -129,7 +137,7 @@ def watch_link_set(
|
||||
admin token, so no separate Plex login. Enabling it for the first time runs the one-time
|
||||
"Plex is master" import immediately so the owner's existing Plex history shows up in Siftlode."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
raise HTTPException(status_code=409, detail="The Plex module is disabled.")
|
||||
raise refused(409, "plex_disabled")
|
||||
enabled = body.enabled
|
||||
link = db.execute(select(PlexLink).where(PlexLink.user_id == user.id)).scalars().first()
|
||||
if link is None:
|
||||
@@ -141,7 +149,7 @@ def watch_link_set(
|
||||
result: dict = {}
|
||||
if enabled and not link.initial_import_done:
|
||||
result = plex_watch.import_owner_watch_state(db, link)
|
||||
return {**_link_dict(link), "import": result or None}
|
||||
return {"ok": True, **_link_dict(link), "import": result or None}
|
||||
|
||||
|
||||
@router.post("/watch/import")
|
||||
@@ -150,6 +158,6 @@ def watch_import_now(user: User = Depends(admin_user), db: Session = Depends(get
|
||||
intersection; Siftlode-only states are preserved. Idempotent."""
|
||||
link = db.execute(select(PlexLink).where(PlexLink.user_id == user.id)).scalars().first()
|
||||
if link is None or not link.sync_enabled:
|
||||
raise HTTPException(status_code=409, detail="Enable Plex watch-sync first.")
|
||||
raise refused(409, "watch_sync_off")
|
||||
result = plex_watch.import_owner_watch_state(db, link)
|
||||
return {**_link_dict(link), "import": result}
|
||||
return {"ok": True, **_link_dict(link), "import": result}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user