Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8142484347 | ||
|
|
0d91b82f86 | ||
|
|
90fc86d437 | ||
|
|
2fe58073e4 | ||
|
|
bc03e82687 | ||
|
|
3f983e6d76 | ||
|
|
134ec9a68e | ||
|
|
9cf177b198 | ||
|
|
a5fae489e0 | ||
|
|
8d0083d4e4 | ||
|
|
9f177557d2 | ||
|
|
46fce4a767 | ||
|
|
2d9595b0e5 | ||
|
|
a7d7456279 | ||
|
|
44a792c34f | ||
|
|
3987cd4260 | ||
|
|
95d9ed4aa6 | ||
|
|
8d0063d514 | ||
|
|
0ca0cac534 | ||
|
|
cf0c55ba80 | ||
|
|
fa4f823fc8 | ||
|
|
884060891e | ||
|
|
955c8c3445 | ||
|
|
99df44d7fb | ||
|
|
30fbdc52b2 | ||
|
|
046ff735cf | ||
|
|
7afaba0547 | ||
|
|
db8ed5a004 | ||
|
|
0ef25a8037 | ||
|
|
17a26894b0 | ||
|
|
946c8f26db | ||
|
|
8759b5860d | ||
|
|
dc902d2c0e | ||
|
|
cedcbb1434 | ||
|
|
4f72b193aa | ||
|
|
7b94e9a651 | ||
|
|
c6bf84a578 | ||
|
|
444f9b8166 | ||
|
|
7a47223a58 | ||
|
|
bc138e3b37 | ||
|
|
df7c3db085 | ||
|
|
6fda49369f | ||
|
|
8018cd4685 | ||
|
|
29875b7367 | ||
|
|
7ae5c57b6b | ||
|
|
dc4e8b9fcd | ||
|
|
6f9ba428d5 | ||
|
|
c60359bfed | ||
|
|
e6b76e156a | ||
|
|
e74986cc9a | ||
|
|
2df45b27d3 | ||
|
|
bacd6aec4f | ||
|
|
fad301e7a4 | ||
|
|
1cfb19bdc7 | ||
|
|
90c30d596f | ||
|
|
7ed6ccf171 | ||
|
|
ce30b99327 | ||
|
|
a7891d03c5 | ||
|
|
038da5374f | ||
|
|
92979fa920 | ||
|
|
2d5bd275fa | ||
|
|
49fe56b6e7 | ||
|
|
1fa6a0bea8 | ||
|
|
582f20fab6 | ||
|
|
6cb8231a9f | ||
|
|
6bbf532b70 | ||
|
|
2fa5c8c394 | ||
|
|
373415a2eb | ||
|
|
582c033e1c | ||
|
|
ef652ee5d8 | ||
|
|
250e3a9f90 | ||
|
|
6096dd6927 | ||
|
|
51e98bfc38 | ||
|
|
3ee00fb051 | ||
|
|
29094f5f51 | ||
|
|
16a44dec79 | ||
|
|
4e3e68d6e6 | ||
|
|
eb29f22eb7 | ||
|
|
8f6f89832f | ||
|
|
1c49870a7a | ||
|
|
142593083f | ||
|
|
8cfab498cf | ||
|
|
b5996a0e56 | ||
|
|
5e208353e3 | ||
|
|
66dbd0af81 | ||
|
|
59fd068711 | ||
|
|
8e12eee996 | ||
|
|
89c77ae6b5 | ||
|
|
0b672ed6e7 | ||
|
|
54d19cc386 | ||
|
|
04b58e5c6f | ||
|
|
6b0401b922 | ||
|
|
0cff964ff2 | ||
|
|
dc221338be | ||
|
|
8c10e57208 | ||
|
|
dd040adc96 | ||
|
|
4ab128ccab | ||
|
|
a28193a00c | ||
|
|
986bc8d01e | ||
|
|
07e3ea3737 | ||
|
|
b2eaa9e00b | ||
|
|
791ba5f507 | ||
|
|
7789db8240 | ||
|
|
f92cd9017f | ||
|
|
4bfba4aa86 | ||
|
|
86bc008a81 | ||
|
|
fac5c4465a | ||
|
|
9c9b1cf6d5 | ||
|
|
c3d88d9b20 | ||
|
|
569190480d | ||
|
|
5883ea509d | ||
|
|
a210992bfc | ||
|
|
0517de1f59 | ||
|
|
b59fd38a9c | ||
|
|
88a166e4aa | ||
|
|
07766c43e8 | ||
|
|
bb3c6dec45 | ||
|
|
3cf8f04d73 | ||
|
|
2b56c5c2eb | ||
|
|
c06a456a9a | ||
|
|
864f673ef5 | ||
|
|
6c63df6704 | ||
|
|
a5bcd9fca2 | ||
|
|
618ef812e6 | ||
|
|
e299c66121 | ||
|
|
11aef4ab7a | ||
|
|
4e52bdd4ca | ||
|
|
595490cb98 | ||
|
|
7537882235 | ||
|
|
47d968eeea | ||
|
|
e7f38396a2 | ||
|
|
f569b19c49 | ||
|
|
e795d191c5 | ||
|
|
4ad7183102 | ||
|
|
2c3aaec62d | ||
|
|
70598d47a3 | ||
|
|
5858e7b513 | ||
|
|
5346efcd10 | ||
|
|
6169e32ca5 | ||
|
|
90523284ab | ||
|
|
17bfea8754 | ||
|
|
2a9ddb78ac | ||
|
|
e887233210 | ||
|
|
b17d92d7c2 | ||
|
|
90ee34a1b4 | ||
|
|
118fc70e08 | ||
|
|
636372511b | ||
|
|
1354f5b503 | ||
|
|
98ec1ffcab | ||
|
|
18c11ca437 | ||
|
|
ca5e446fbd | ||
|
|
4c7ab12a3e | ||
|
|
c6b552e75f | ||
|
|
3cd0c525e2 | ||
|
|
f130a1c05a | ||
|
|
bbd31c7143 | ||
|
|
8731508645 | ||
|
|
b5d37ca142 | ||
|
|
eb0507f942 | ||
|
|
6d09b94bbb | ||
|
|
5274216981 | ||
|
|
75d89613d0 | ||
|
|
8fb276aa90 | ||
|
|
78a9324381 | ||
|
|
8a50538275 | ||
|
|
b4aea7fa5b | ||
|
|
92c305bab9 | ||
|
|
32fe0768b6 | ||
|
|
415a6c16f9 | ||
|
|
8c51062e5b | ||
|
|
63bf1bca78 | ||
|
|
c7a377396b | ||
|
|
8036251589 | ||
|
|
71d7646bf1 | ||
|
|
3eb0f41962 | ||
|
|
a0114deff1 | ||
|
|
d89d887a9b | ||
|
|
f8a31d48cf | ||
|
|
7f17b17c13 | ||
|
|
1f81684b6e | ||
|
|
33fddacee2 | ||
|
|
12121255dd | ||
|
|
4a7b65d187 | ||
|
|
47af4d5c05 | ||
|
|
8a83bc0e9f | ||
|
|
41b652f8fd | ||
|
|
545996f807 | ||
|
|
267fd7f70f | ||
|
|
6d49da17fe | ||
|
|
6a92585098 | ||
|
|
31dc4184f5 | ||
|
|
86971c574d | ||
|
|
785080f3da | ||
|
|
76b5f7c93e | ||
|
|
1cfe6a7f59 | ||
|
|
33fe895539 | ||
|
|
5c5cabbe5a | ||
|
|
4f37c41021 | ||
|
|
f7faa19ee2 | ||
|
|
0b4b268678 | ||
|
|
d349bc9084 | ||
|
|
dcd8a9566a | ||
|
|
6de2cd950d | ||
|
|
645d14f45d | ||
|
|
7360339039 | ||
|
|
473d9f02f2 | ||
|
|
d2e225666b | ||
|
|
efd95fca01 | ||
|
|
060deb89d2 | ||
|
|
4869d6c108 | ||
|
|
d20317e0f1 | ||
|
|
d2ba65e383 | ||
|
|
cf7ec5e9ec | ||
|
|
f76a2afbf1 | ||
|
|
889d2acf52 | ||
|
|
782362b808 | ||
|
|
5a42d76e8a | ||
|
|
ed4ab4e13c | ||
|
|
b8bff504d8 | ||
|
|
1630a3df3c | ||
|
|
14b5fe2e31 | ||
|
|
d260aabfd7 | ||
|
|
d832bb19ce | ||
|
|
59efc8f2e8 | ||
|
|
e4ffd59ca7 | ||
|
|
3709b71a94 | ||
|
|
cedebe0ffb | ||
|
|
fdc1eabbe0 | ||
|
|
84f4b20e5a | ||
|
|
828dad015c | ||
|
|
82914b56ca | ||
|
|
ccd5f3739a | ||
|
|
f3c0b7be23 | ||
|
|
53b50231ff | ||
|
|
5703c4d713 | ||
|
|
013f19faca | ||
|
|
8d9a99ce54 | ||
|
|
45dd4bdda7 | ||
|
|
dac391a890 | ||
|
|
4b33036e7c | ||
|
|
80dea4f307 | ||
|
|
20e17c0ccb | ||
|
|
7d302f71b7 | ||
|
|
f99240d8ca | ||
|
|
cc8cf96c19 | ||
|
|
86163958d6 | ||
|
|
12822a9e04 | ||
|
|
928eac4c89 | ||
|
|
b28be682fe | ||
|
|
7e97559615 | ||
|
|
721467ef69 | ||
|
|
47e2293de3 | ||
|
|
29bb914a95 | ||
|
|
5bfcf5d83a | ||
|
|
31fa554b87 | ||
|
|
6a0efba477 | ||
|
|
9f28f46839 | ||
|
|
14f1c97c0a | ||
|
|
fc56f514b3 | ||
|
|
358d722ce9 | ||
|
|
fe38dfbcca | ||
|
|
f98b060e2b | ||
|
|
bc9e34d0ae | ||
|
|
0cc450d1c9 | ||
|
|
688efb9ed6 | ||
|
|
ca4e559cc7 | ||
|
|
e23128b4a1 | ||
|
|
0d5f8f0708 | ||
|
|
a03bf8d371 | ||
|
|
3dc14402b0 | ||
|
|
540ca244c8 | ||
|
|
563fa0513d | ||
|
|
10c0ca4cf7 | ||
|
|
a76bab9b34 | ||
|
|
7acdf5d577 | ||
|
|
b1086e05dc | ||
|
|
ed91c83dd7 | ||
|
|
a614ee2435 | ||
|
|
c74487fbb3 | ||
|
|
876cece881 | ||
|
|
e5997ca567 | ||
|
|
c134b514c1 | ||
|
|
243869401a | ||
|
|
7339a0b84b | ||
|
|
c960c07a6a | ||
|
|
3cfdb36f06 | ||
|
|
ada95d04a4 | ||
|
|
1b004e6ccf | ||
|
|
95c33e0b0f | ||
|
|
4e6bcbc078 | ||
|
|
e17fb5bbf5 | ||
|
|
a9044ece50 | ||
|
|
473aafd1c8 | ||
|
|
7019e3afdf | ||
|
|
c81b110637 | ||
|
|
2ecc70ca6d | ||
|
|
f8c63e80c2 | ||
|
|
3ac3d382e7 | ||
|
|
94c62ca699 | ||
|
|
1fe24d374a | ||
|
|
2cba6b9fca | ||
|
|
8d7aaad9a8 | ||
|
|
85043013f0 | ||
|
|
44f626f732 | ||
|
|
24c93cbb83 | ||
|
|
cb1f4f00b0 | ||
|
|
f6e1d1e69f | ||
|
|
b64c3f514d | ||
|
|
b5a1002412 | ||
|
|
ea931dca30 | ||
|
|
d954e71a77 | ||
|
|
602f62967a | ||
|
|
ba00b50dea | ||
|
|
92adb593c8 | ||
|
|
66132b6dda | ||
|
|
d09f021f0c | ||
|
|
d7e9240e54 | ||
|
|
f2fa49b778 | ||
|
|
d4ed45b2eb | ||
|
|
375f6e58be | ||
|
|
1a102ce81a | ||
|
|
bfd45a155e | ||
|
|
1102ad6a28 | ||
|
|
c1a3b7b935 | ||
|
|
bf98b48517 | ||
|
|
b563155a37 | ||
|
|
915319fa35 | ||
|
|
a2a8f5681d | ||
|
|
4827aebb7f | ||
|
|
840526b1b8 | ||
|
|
10d69cb316 | ||
|
|
a86131e062 | ||
|
|
a84f2db310 | ||
|
|
872d91f540 | ||
|
|
4eaa68ceeb | ||
|
|
00539a4d97 | ||
|
|
dc24d353e0 | ||
|
|
ce96032edb | ||
|
|
0658f9550b | ||
|
|
f162788135 | ||
|
|
968674bc6a | ||
|
|
90ae9f6e88 | ||
|
|
3682e6a01f | ||
|
|
aca355f43b | ||
|
|
f41c81eb56 | ||
|
|
614c5528ad | ||
|
|
104bd1e3a6 | ||
|
|
584e9ced0c | ||
|
|
b4ad59274b | ||
|
|
fe0b34cd00 | ||
|
|
9eaa01617c | ||
|
|
fdca12968b | ||
|
|
94fcc40bde | ||
|
|
fb2c843fa9 | ||
|
|
bb6cb6e335 | ||
|
|
bdbff89507 | ||
|
|
9b87838e3a | ||
|
|
f014416e31 | ||
|
|
3369587412 | ||
|
|
9af4e8f4ff | ||
|
|
a0b725d779 | ||
|
|
bc33b2ed16 | ||
|
|
5280949667 | ||
|
|
4892c38466 | ||
|
|
9ae6c47d6c | ||
|
|
b7a6451dba | ||
|
|
11e32163b8 | ||
|
|
257dc5741d | ||
|
|
f52bc5898b | ||
|
|
d0033617e2 | ||
|
|
f80ed115ad | ||
|
|
7fbb996297 | ||
|
|
9cd4ae09d4 | ||
|
|
00527d0cf3 | ||
|
|
8dc4976c2c | ||
|
|
56a2ecdec3 | ||
|
|
8299ea72b7 | ||
|
|
6c918d1dff | ||
|
|
3f8257059d | ||
|
|
12d9561ba1 | ||
|
|
fe09879edd | ||
|
|
7104767ff9 | ||
|
|
17f8885fb1 | ||
|
|
549aae67cc | ||
|
|
fbb961b5b7 | ||
|
|
edcea87ea5 | ||
|
|
2c843e549b | ||
|
|
4ee0dab68f | ||
|
|
5d7be687d3 |
@@ -28,6 +28,14 @@ ALLOWED_EMAILS=
|
||||
# Admin emails (subset of the above) get the admin role.
|
||||
ADMIN_EMAILS=
|
||||
|
||||
# Logging (E22). LOG_LEVEL is the BOOTSTRAP level — the level a process starts at and the only one
|
||||
# the worker/download child read; the API's running level is the admin Configuration page's
|
||||
# `log_level` knob, which takes effect without a restart. TRACE | DEBUG | INFO | WARNING | ERROR, in
|
||||
# any letter case; any other value stops the boot with an error. LOG_FORMAT is "json" (one object per
|
||||
# line — what a log collector ingests) or "text" (a readable line for a local terminal).
|
||||
LOG_LEVEL=INFO
|
||||
LOG_FORMAT=json
|
||||
|
||||
# Optional: origin of a separately-served frontend dev server (enables CORS). Leave empty in production.
|
||||
FRONTEND_ORIGIN=
|
||||
|
||||
|
||||
+2106
File diff suppressed because it is too large
Load Diff
@@ -26,8 +26,8 @@ The local gate is **ruff (`check` + `format --check`) + typecheck (3 tsc project
|
||||
keep commits logical and Conventional-Commit-styled.
|
||||
- **No AI attribution in commits.** No `Co-Authored-By` trailer, no "generated by" lines. (This file
|
||||
being AI-written is the intended, visible exception; commits are not.)
|
||||
- **Release: user-owned in timing, agent-run in execution.** The user decides *when* to ship; on their
|
||||
go-ahead *you* run `siftlode release` from the feature branch (merge to `main` → publish → deploy →
|
||||
- **Release: user-owned in timing, agent-run in execution.** The user decides _when_ to ship; on their
|
||||
go-ahead _you_ run `siftlode release` from the feature branch (merge to `main` → publish → deploy →
|
||||
verify `/api/version`). Never self-ship — bring work to a green, committed state and stop for the
|
||||
go-ahead first, then run it yourself.
|
||||
- **Public repo.** Keep ops/topology/secrets out — those live in a separate private repo. Nothing here
|
||||
@@ -41,7 +41,7 @@ how it gets built. Concretely:
|
||||
- **Plan before you build.** For anything non-trivial, present a short plan (sprint-level for larger
|
||||
work) and get a go-ahead before writing code.
|
||||
- **Offer choices, mark the optimal — and explain the trade-off.** When several solutions are viable,
|
||||
list them and mark the recommended one. This is the *one* place to spend extra words: give the *why*
|
||||
list them and mark the recommended one. This is the _one_ place to spend extra words: give the _why_
|
||||
behind each option and the weight of the decision. These decision points are where the product owner
|
||||
engages with the build and learns from it, so make them understandable — a reasoned menu, not a bare one.
|
||||
- **Mockups for design/UX decisions.** For any visual or interaction choice, show mockup options to
|
||||
@@ -56,34 +56,47 @@ how it gets built. Concretely:
|
||||
## Architecture shape (don't fight it)
|
||||
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`,
|
||||
`security`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → components; data
|
||||
via react-query (`qk` factory), per-account `localStorage`, and server prefs; API through `lib/api/*`
|
||||
slices over `core.req<T>`.
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||
its menus, `Modal`/`Overlay`, `Button`/`form`, `DataTable`/`VirtualGrid`, `QueryState`, …). That
|
||||
boundary is one-way and **lint-enforced**, not just described: a platform component may not import a
|
||||
feature component or a domain API slice, and one layer down `lib/` may not import from `components/` at
|
||||
all, not even a type (`no-restricted-imports` in `eslint.config.js`, with its own test). A new generic
|
||||
component goes in `ui/`; see [`frontend/src/components/ui/README.md`](frontend/src/components/ui/README.md).
|
||||
Data via react-query (`qk` factory), per-account `localStorage`, and server prefs; API through
|
||||
`lib/api/*` slices over `core.req<T>`.
|
||||
|
||||
**Not wanted** (reviewed and rejected as over-engineering for this size): microservices, event bus,
|
||||
CQRS, a repository/DI framework, a router library, a generic CRUD-router factory or base serializer.
|
||||
|
||||
## Canonical patterns — when you have a choice, pick THIS
|
||||
|
||||
| Area | Use | Not |
|
||||
|---|---|---|
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`) | ad-hoc `{"unsubscribed"}`/`{"cleared": n}`/echo |
|
||||
| Transaction commit | the handler/job-wrapper commits; helpers `flush` | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE control | the chip/segmented primitives (`Select` only for >6 options) | a native `<select>` |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| Area | Use | Not |
|
||||
| ----------------------- | ------------------------------------------------------------ | ------------------------------------------------------ |
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE action button | `ui/Button` — pick a `SIZES` rung | a hand-rolled `<button className="px-3 py-2 …">` |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
|
||||
## House rules that bite (learned from real bugs)
|
||||
|
||||
@@ -100,6 +113,6 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
|
||||
---
|
||||
|
||||
*Run-it-locally and the full gate commands live in [`docs/development.md`](docs/development.md); backend
|
||||
_Run-it-locally and the full gate commands live in [`docs/development.md`](docs/development.md); backend
|
||||
code conventions in [`backend/app/README.md`](backend/app/README.md). This file stays the source of
|
||||
truth for the canonical-patterns table and the gate/workflow/collaboration rules above.*
|
||||
truth for the canonical-patterns table and the gate/workflow/collaboration rules above._
|
||||
|
||||
+20
-4
@@ -45,18 +45,34 @@ RUN adduser --disabled-password --gecos "" appuser \
|
||||
&& mkdir -p /downloads \
|
||||
&& chown appuser /app /downloads
|
||||
|
||||
# ffmpeg: merge separate video+audio streams + postprocessing. Deno: the JavaScript runtime
|
||||
# yt-dlp uses to solve YouTube's "n" signature challenge for the web player clients (required
|
||||
# alongside the PO token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
||||
# ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode (E6 S1).
|
||||
# intel-media-va-driver (iHD): the VA driver ffmpeg's h264_vaapi needs — libva2 ships in the base but
|
||||
# no driver, so without this `vainfo`/`h264_vaapi` fail and transcode silently falls back to libx264.
|
||||
# Harmless on hosts with no /dev/dri (the driver just goes unused). Deno: the JavaScript runtime yt-dlp
|
||||
# uses to solve YouTube's "n" signature challenge for the web player clients (required alongside the PO
|
||||
# token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
||||
ARG DENO_VERSION=v2.9.1
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ffmpeg unzip curl ca-certificates \
|
||||
&& apt-get install -y --no-install-recommends ffmpeg intel-media-va-driver unzip curl ca-certificates \
|
||||
&& curl -fsSL "https://github.com/denoland/deno/releases/download/${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o /tmp/deno.zip \
|
||||
&& unzip -o /tmp/deno.zip -d /usr/local/bin \
|
||||
&& chmod +x /usr/local/bin/deno \
|
||||
&& rm /tmp/deno.zip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Intel NEO OpenCL runtime (intel-opencl-icd) — needed by the Plex transcode's HDR/10-bit HW path, which
|
||||
# tone-maps HDR→SDR on the GPU via `tonemap_opencl` (Gen 9.5 / UHD 630 can't tonemap in the VAAPI VPP, so
|
||||
# OpenCL is the only HW route; without this the 10-bit branch drops to CPU libx264). NEO was REMOVED from
|
||||
# Debian trixie (this base = python:3.12-slim = trixie), so it's pulled from **bookworm** — the 22.43 there
|
||||
# still supports Gen 9.5 and its .deb runs on trixie (verified live on the prod UHD 630). The bookworm apt
|
||||
# source is added only for this one install and removed after, so it can't drift the rest of the image.
|
||||
# Harmless on hosts with no /dev/dri or no OpenCL use (the runtime just sits unused).
|
||||
RUN printf 'Types: deb\nURIs: http://deb.debian.org/debian\nSuites: bookworm\nComponents: main\nSigned-By: /usr/share/keyrings/debian-archive-keyring.pgp\n' > /etc/apt/sources.list.d/bookworm.sources \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends -t bookworm intel-opencl-icd \
|
||||
&& rm /etc/apt/sources.list.d/bookworm.sources \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install from the PINNED lock (requirements.lock, generated from requirements.txt) so the image is
|
||||
# byte-reproducible and the exact versions are the ones pip-audit scanned in CI — not whatever floats
|
||||
# up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
||||
|
||||
@@ -0,0 +1,661 @@
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
@@ -94,9 +94,18 @@ docker compose -f docker-compose.selfhost.yml pull # or: docker compose pull
|
||||
docker compose -f docker-compose.selfhost.yml up -d
|
||||
```
|
||||
|
||||
Database migrations run automatically on startup. Your data (accounts, subscriptions, playlists, the
|
||||
video catalog) lives in a Postgres volume — back it up with `scripts/backup.sh` (or `backup.ps1` on
|
||||
Windows) and restore with `scripts/restore.sh`.
|
||||
That follows `latest`, which moves with every release. Set `IMAGE_TAG=0.80.0` in `.env` to pin a
|
||||
version and upgrade on your own schedule; released versions are listed in [`CHANGELOG.md`](CHANGELOG.md)
|
||||
and in the app's Release Notes. The registry keeps only the newest few of those, so a pin decides
|
||||
*when* you take the next hop rather than letting you stay put forever — see
|
||||
[docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
||||
|
||||
**Back up before upgrading.** Migrations run automatically on startup and are not undone by pulling
|
||||
the old image back. A backup is the database dump **plus your `.env`** — `TOKEN_ENCRYPTION_KEY` is
|
||||
what decrypts the Google, SMTP and Plex secrets stored in that database, so a dump restored without it
|
||||
is only half an instance. Use `scripts/backup.sh` (or `backup.ps1` on Windows) and `scripts/restore.sh`;
|
||||
the equivalent one-liners, the pinning recipe and the rollback steps are in
|
||||
[docs/self-hosting.md](docs/self-hosting.md#backups).
|
||||
|
||||
## How it works
|
||||
|
||||
@@ -111,6 +120,12 @@ Windows) and restore with `scripts/restore.sh`.
|
||||
FastAPI + PostgreSQL (SQLAlchemy, Alembic) backend; React + Vite + Tailwind + TanStack Query
|
||||
frontend; packaged as a single Docker image with Docker Compose.
|
||||
|
||||
## License
|
||||
|
||||
[GNU AGPL-3.0](LICENSE). You can run, study, modify and share Siftlode; if you modify it and let other
|
||||
people use it over a network, they are entitled to your modified source. Copyright © 2026 Siftlode
|
||||
contributors.
|
||||
|
||||
## Note
|
||||
|
||||
This project is developed with AI assistance.
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN adduser --disabled-password --gecos "" appuser && chown -R appuser /app
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8000
|
||||
CMD ["sh", "entrypoint.sh"]
|
||||
@@ -1,6 +1,8 @@
|
||||
[alembic]
|
||||
script_location = alembic
|
||||
prepend_sys_path = .
|
||||
# Split the path list the way the OS does (alembic's legacy space/comma/colon split is deprecated).
|
||||
path_separator = os
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
@@ -1,17 +1,17 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from app.config import settings
|
||||
from app.db import Base
|
||||
from app.logs import configure_logging
|
||||
import app.models # noqa: F401 (import so models register on Base.metadata)
|
||||
|
||||
config = context.config
|
||||
config.set_main_option("sqlalchemy.url", settings.database_url)
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
# Migrations run at container start (entrypoint.sh), so their output is part of the API's log
|
||||
# stream — one format with everything else, not alembic.ini's own fileConfig shape.
|
||||
configure_logging("alembic")
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""download_credentials (S6: per-user, per-host yt-dlp cookie/auth store)
|
||||
|
||||
Each user stores their OWN login cookies (Netscape cookies.txt) for an auth-walled download host
|
||||
(Facebook, Instagram, …), Fernet-encrypted at rest. A download of a yt-dlp-supported, auth-walled
|
||||
link on that host is retried with the acting user's cookie. Unique per (user_id, host_key) so a
|
||||
user holds at most one credential per host. Empty for every existing install; forward-only in
|
||||
practice (a downgrade just drops the table + its secrets).
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0074_download_credentials"
|
||||
down_revision: Union[str, None] = "0073_saved_view_source"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"download_credentials",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("host_key", sa.String(length=64), nullable=False),
|
||||
sa.Column("cookies_enc", sa.Text(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.UniqueConstraint("user_id", "host_key", name="uq_download_credential"),
|
||||
)
|
||||
op.create_index("ix_download_credentials_user_id", "download_credentials", ["user_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_download_credentials_user_id", table_name="download_credentials")
|
||||
op.drop_table("download_credentials")
|
||||
@@ -0,0 +1,26 @@
|
||||
"""playlist synced_video_ids (S7: 3-way merge baseline for two-way YouTube sync)
|
||||
|
||||
The one-directional "local wins" push is replaced by a 3-way merge that classifies each item
|
||||
against the last-synced state. That needs the raw ordered id set as a baseline, not just the
|
||||
`synced_fingerprint` hash. Add a nullable `synced_video_ids` (JSON list) column; NULL on every
|
||||
existing row means "no baseline yet" — the first merge then unions losslessly and records one.
|
||||
Forward-only in practice (a downgrade just drops the column).
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0075_playlist_sync_baseline"
|
||||
down_revision: Union[str, None] = "0074_download_credentials"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("playlists", sa.Column("synced_video_ids", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("playlists", "synced_video_ids")
|
||||
@@ -0,0 +1,41 @@
|
||||
"""media_assets.media_kind (E7 S5a: the video|audio|file discriminator for the media library)
|
||||
|
||||
Adds a stored media-kind to each asset so the library can gate its actions on it (play/edit for
|
||||
media, download-only for a generic uploaded file) instead of re-deriving from codecs in prose. The
|
||||
column is the single home of the invariant `downloads.mediakind.derive` computes at finalize.
|
||||
|
||||
Backfill for existing rows: a stored video codec -> 'video', an audio-only codec -> 'audio', an
|
||||
editor derivative (source_kind='edit', which carries no codecs but is always a video clip) ->
|
||||
'video'. Rows still pending/errored (no codecs, not an edit) stay NULL — kind is set when they
|
||||
finalize. Empty `media_kind` never reaches the UI: only `ready`/`done` rows render actions, and
|
||||
those are all backfilled here.
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0076_media_asset_kind"
|
||||
down_revision: Union[str, None] = "0075_playlist_sync_baseline"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("media_assets", sa.Column("media_kind", sa.String(length=16), nullable=True))
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE media_assets
|
||||
SET media_kind = CASE
|
||||
WHEN vcodec IS NOT NULL THEN 'video'
|
||||
WHEN acodec IS NOT NULL THEN 'audio'
|
||||
WHEN source_kind = 'edit' THEN 'video'
|
||||
ELSE NULL
|
||||
END
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("media_assets", "media_kind")
|
||||
@@ -0,0 +1,54 @@
|
||||
"""plex_items playback-class facet (E7: filter the feed by Direct Play / Direct Stream / Transcode)
|
||||
|
||||
Adds the columns the playback-class filter needs:
|
||||
* transcode_hw — for a `playable="transcode"` item, whether the prod iGPU can hardware-transcode it
|
||||
(codec HW-decodable AND 8-bit SDR) vs software-only. NULL for non-transcode items.
|
||||
* video_bit_depth — the source video bit depth (from the Plex media Stream), NULL until probed.
|
||||
* media_probed_at — watermark for the bit-depth backfill (NULL = not yet probed).
|
||||
|
||||
Backfill for existing rows: `transcode_hw` is derived from the ALREADY-stored `codec_video` alone (the
|
||||
same HW-decodable allow-list `plex.sync.HW_DECODABLE_CODECS` / `plex.stream` use), so the filter works
|
||||
immediately after migration. Bit depth is unknown for existing rows (the cheap Plex listing doesn't
|
||||
carry it), so a 10-bit/HDR HEVC is provisionally 'hw' here and later refined to 'sw' by the bit-depth
|
||||
backfill (`plex.sync._probe_media_batch`) once its Stream is probed. `video_bit_depth`/`media_probed_at`
|
||||
start NULL.
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0077_plex_playback_class"
|
||||
down_revision: Union[str, None] = "0076_media_asset_kind"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("plex_items", sa.Column("transcode_hw", sa.Boolean(), nullable=True))
|
||||
op.add_column("plex_items", sa.Column("video_bit_depth", sa.Integer(), nullable=True))
|
||||
op.add_column(
|
||||
"plex_items", sa.Column("media_probed_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
op.create_index("ix_plex_items_transcode_hw", "plex_items", ["transcode_hw"])
|
||||
# Codec-only derivation for existing rows — keep this codec set identical to
|
||||
# plex.sync.HW_DECODABLE_CODECS. Bit depth is unknown here, so provisionally 8-bit (→ hw for a
|
||||
# decodable codec); the bit-depth backfill downgrades 10-bit/HDR HEVC to sw later.
|
||||
op.execute(
|
||||
"""
|
||||
UPDATE plex_items
|
||||
SET transcode_hw = CASE
|
||||
WHEN playable = 'transcode'
|
||||
THEN lower(codec_video) IN ('h264','hevc','h265','vp9','vc1','mpeg2video')
|
||||
ELSE NULL
|
||||
END
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_plex_items_transcode_hw", table_name="plex_items")
|
||||
op.drop_column("plex_items", "media_probed_at")
|
||||
op.drop_column("plex_items", "video_bit_depth")
|
||||
op.drop_column("plex_items", "transcode_hw")
|
||||
@@ -0,0 +1,58 @@
|
||||
"""users.password_revoked_at — remember that SEC-10 dropped a preset password
|
||||
|
||||
Revision ID: 0078_password_revoked_at
|
||||
Revises: 0077_plex_playback_class
|
||||
Create Date: 2026-09-02
|
||||
|
||||
SEC-10 (`auth.revoke_preset_password_if_unverified`) nulls `password_hash` when an account whose email
|
||||
was never verified is attested (Google adoption, or admin approval). That left NO trace, and
|
||||
password-reset eligibility keyed off `password_hash IS NOT NULL` — so such an account could neither
|
||||
sign in (no password) nor request a reset link (not "eligible"): a permanent lockout with a silent,
|
||||
uniform response at every step.
|
||||
|
||||
This column is that trace. A row with `password_revoked_at` set is a local account that HAD a password
|
||||
and had it revoked, so the reset flow treats it as eligible again. Setting a password (reset-confirm or
|
||||
/auth/set-password) clears the stamp. A Google-only account that never had a password keeps it NULL and
|
||||
stays reset-ineligible by design — its owner adds a password from Settings while signed in.
|
||||
|
||||
Backfill: the signature is exact and narrow — a local (google_sub IS NULL), non-demo account with NO
|
||||
password could ONLY have got there through the revoke. Registration always sets a hash, the setup
|
||||
wizard sets one too, every Google row carries a google_sub (the adoption path assigns it BEFORE
|
||||
revoking), and nothing in the codebase ever clears google_sub. Stamp those with the current time so
|
||||
they become recoverable without operator intervention.
|
||||
|
||||
Deliberately NOT also requiring `email_verified = false`, even though the revoke only fires on an
|
||||
unverified account: verifying is exactly what such a victim does NEXT. The revoke does not invalidate
|
||||
the outstanding verification token, so clicking the emailed link sets email_verified=True while
|
||||
password_hash stays NULL — and that row has no route back at all (reset ineligible, resend refuses an
|
||||
already-verified address, and set-password needs a signed-in session the account can no longer obtain:
|
||||
no password to sign in with, and no Google identity). Those are the victims who most need the stamp,
|
||||
so the predicate must not exclude them.
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0078_password_revoked_at"
|
||||
down_revision: Union[str, None] = "0077_plex_playback_class"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"users",
|
||||
sa.Column("password_revoked_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.execute(
|
||||
"UPDATE users SET password_revoked_at = now() "
|
||||
"WHERE password_hash IS NULL "
|
||||
"AND google_sub IS NULL "
|
||||
"AND is_demo = false"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "password_revoked_at")
|
||||
@@ -0,0 +1,65 @@
|
||||
"""email_canonical on users + invites, for matching one mailbox across Gmail spellings
|
||||
|
||||
Revision ID: 0079_email_canonical
|
||||
Revises: 0078_password_revoked_at
|
||||
Create Date: 2026-09-04
|
||||
|
||||
Gmail delivers `x.y@gmail.com`, `xy@gmail.com` and `xy+tag@gmail.com` to ONE mailbox; the invite gate
|
||||
only lowercased. On 2026-09-03 that turned an approved user's Google sign-in into a silent second
|
||||
access request (OAuth returns the account's stored primary, which need not be the spelling they typed)
|
||||
and left prod with two user rows for one mailbox.
|
||||
|
||||
The backfill below reproduces `app.utils.canonical_email` in SQL. That duplication is deliberate — a
|
||||
migration must not import application code that will keep changing — but it means the two have to
|
||||
agree; `test_canonical_email_backfill_matches_the_helper` pins them together.
|
||||
|
||||
Deliberately NOT unique. Prod already carries canonical duplicates: two approved invites for one
|
||||
mailbox dating back to 2026-06-16, and two active user rows from the 2026-09-03 walkthrough. Merging
|
||||
accounts is a judgement call about people's data, not something a migration gets to do on its own, so
|
||||
this migration only records the canonical form and leaves the duplicates in place for a human.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0079_email_canonical"
|
||||
down_revision: str | None = "0078_password_revoked_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
# Mirrors app.utils.canonical_email: Gmail (and its googlemail alias) loses dots and any `+tag` from
|
||||
# the local part and lands on gmail.com; every other domain is only lowercased, because the dot rule
|
||||
# is not guaranteed elsewhere — and does not hold for Workspace custom domains unless their admin
|
||||
# enabled it.
|
||||
_CANON_SQL = """
|
||||
CASE
|
||||
WHEN split_part(lower({col}), '@', 2) IN ('gmail.com', 'googlemail.com')
|
||||
-- an empty local part after stripping falls through to plain lowercase, as the helper does
|
||||
AND replace(split_part(split_part(lower({col}), '@', 1), '+', 1), '.', '') <> ''
|
||||
THEN replace(split_part(split_part(lower({col}), '@', 1), '+', 1), '.', '') || '@gmail.com'
|
||||
ELSE lower({col})
|
||||
END
|
||||
"""
|
||||
|
||||
|
||||
def _backfill(table: str) -> None:
|
||||
op.execute(
|
||||
f"UPDATE {table} SET email_canonical = {_CANON_SQL.format(col='email')} "
|
||||
f"WHERE email IS NOT NULL"
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for table in ("users", "invites"):
|
||||
op.add_column(
|
||||
table,
|
||||
sa.Column("email_canonical", sa.String(length=320), nullable=False, server_default=""),
|
||||
)
|
||||
_backfill(table)
|
||||
op.create_index(f"ix_{table}_email_canonical", table, ["email_canonical"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table in ("users", "invites"):
|
||||
op.drop_index(f"ix_{table}_email_canonical", table_name=table)
|
||||
op.drop_column(table, "email_canonical")
|
||||
@@ -0,0 +1,32 @@
|
||||
"""users.google_email — the address Google asserts, kept beside the one the user registered with
|
||||
|
||||
Revision ID: 0080_google_email
|
||||
Revises: 0079_email_canonical
|
||||
Create Date: 2026-09-04
|
||||
|
||||
Since 0079's follow-up a Google sign-in no longer overwrites `users.email`, so a deliberate
|
||||
`me+siftlode@gmail.com` survives being linked to a Google account whose primary is `me@gmail.com`.
|
||||
That leaves two true addresses for one account and only one of them stored. This column holds the
|
||||
other, so the account menu and the Settings sign-in row can show which address Google actually
|
||||
answers with.
|
||||
|
||||
No backfill is possible: Google's answer is only known at sign-in. Accounts linked before this
|
||||
column existed carry NULL until their owner next signs in with Google — and for the majority the two
|
||||
addresses are identical anyway, where the UI shows nothing either way.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0080_google_email"
|
||||
down_revision: str | None = "0079_email_canonical"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("users", sa.Column("google_email", sa.String(length=320), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "google_email")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""invites.invited_at — when the admin actually told this person the door was open
|
||||
|
||||
Revision ID: 0081_invite_invited_at
|
||||
Revises: 0080_google_email
|
||||
Create Date: 2026-09-05
|
||||
|
||||
Whitelisting an address stays SILENT by design: the operator is opening a door as a possibility,
|
||||
usually after agreeing it with the person, and a row can sit approved for weeks without anyone
|
||||
registering. Telling them is therefore a separate, deliberate act — a send button on the row.
|
||||
|
||||
Which raises the one question the admin cannot otherwise answer: "did I already write to them?"
|
||||
Without a record the only failure mode is writing to a real person twice, which is exactly the
|
||||
nudging the silent-by-default design exists to avoid. The audit log holds the event, but not where
|
||||
the decision is made.
|
||||
|
||||
NULL means "never invited from here", which is true of every existing row: no backfill is possible
|
||||
or wanted, since the historic approval mails answered a REQUEST rather than inviting anyone.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0081_invite_invited_at"
|
||||
down_revision: str | None = "0080_google_email"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("invites", sa.Column("invited_at", sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("invites", "invited_at")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""invites.lang — the language this person asked us in
|
||||
|
||||
Revision ID: 0082_invite_lang
|
||||
Revises: 0081_invite_invited_at
|
||||
Create Date: 2026-09-05
|
||||
|
||||
The mails an access request produces — approved, denied, invited — go to an address that often has
|
||||
no account yet, so there is no `preferences.language` to read. The signal does exist, though, and
|
||||
only at the moment of the request: the language of the page the form was submitted from, or the
|
||||
locale Google reported for a sign-in we refused. Recording it there is what lets the decision mail
|
||||
answer in the requester's own language instead of an admin's guess about a stranger.
|
||||
|
||||
Captured rather than derived, for the third time in this epic: `preferences.language` (S6) and
|
||||
`google_email` (S3) both taught that a signal which exists only in one moment cannot be backfilled
|
||||
later. So no backfill here either — NULL means "we were never told", which is the honest state for
|
||||
every historic row, and for every address an admin whitelists by hand. Those fall back to English
|
||||
unless the admin picks otherwise on the invitation send.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0082_invite_lang"
|
||||
down_revision: str | None = "0081_invite_invited_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("invites", sa.Column("lang", sa.String(length=8), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("invites", "lang")
|
||||
@@ -0,0 +1,78 @@
|
||||
"""comfort_pins — the shelf you fall back to when you cannot decide
|
||||
|
||||
Revision ID: 0083_comfort_pins
|
||||
Revises: 0082_invite_lang
|
||||
Create Date: 2026-09-06
|
||||
|
||||
E24's one new table. The shelf holds five different things — a video, a YouTube channel, a Plex show,
|
||||
a Plex movie, a playlist — because comfort media is usually "put on X", a series or a channel, and
|
||||
almost never one particular video.
|
||||
|
||||
Three shapes were possible and two are wrong here:
|
||||
|
||||
- a boolean on `video_states` / `plex_states` cannot ORDER a shelf, and has nowhere to put a channel
|
||||
or a show (neither has a per-user state row at all);
|
||||
- a polymorphic `(kind, ref)` string pair orders and holds anything, but drops the foreign key — and a
|
||||
Plex item removed from the library, or a channel the user unsubscribes and prunes, would then leave
|
||||
a dead tile on the one surface whose entire job is to be reliable when nothing else is.
|
||||
|
||||
So: one nullable CASCADE-ing FK per kind, plus `num_nonnulls(...) = 1`. The union invariant lives in
|
||||
the schema rather than in a comment at every insert site, which is the same rule that the
|
||||
`cancelled`/`canceled` bug taught this codebase.
|
||||
|
||||
NULLs are DISTINCT in a Postgres unique index, so the five per-kind `(user_id, <ref>)` uniques each
|
||||
stop a double-pin without colliding with the other four kinds' NULL rows.
|
||||
|
||||
No backfill: an empty shelf is the honest state for every existing account. The surface renders its
|
||||
own empty state and invites the first pin.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0083_comfort_pins"
|
||||
down_revision: str | None = "0082_invite_lang"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"comfort_pins",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("video_id", sa.String(length=16), sa.ForeignKey("videos.id", ondelete="CASCADE")),
|
||||
sa.Column(
|
||||
"channel_id", sa.String(length=32), sa.ForeignKey("channels.id", ondelete="CASCADE")
|
||||
),
|
||||
sa.Column("plex_item_id", sa.Integer(), sa.ForeignKey("plex_items.id", ondelete="CASCADE")),
|
||||
sa.Column("plex_show_id", sa.Integer(), sa.ForeignKey("plex_shows.id", ondelete="CASCADE")),
|
||||
sa.Column("playlist_id", sa.Integer(), sa.ForeignKey("playlists.id", ondelete="CASCADE")),
|
||||
sa.Column("position", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
"num_nonnulls(video_id, channel_id, plex_item_id, plex_show_id, playlist_id) = 1",
|
||||
name="ck_comfort_pin_one_ref",
|
||||
),
|
||||
sa.UniqueConstraint("user_id", "video_id", name="uq_comfort_pin_video"),
|
||||
sa.UniqueConstraint("user_id", "channel_id", name="uq_comfort_pin_channel"),
|
||||
sa.UniqueConstraint("user_id", "plex_item_id", name="uq_comfort_pin_plex_item"),
|
||||
sa.UniqueConstraint("user_id", "plex_show_id", name="uq_comfort_pin_plex_show"),
|
||||
sa.UniqueConstraint("user_id", "playlist_id", name="uq_comfort_pin_playlist"),
|
||||
)
|
||||
op.create_index("ix_comfort_pins_user_id", "comfort_pins", ["user_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_comfort_pins_user_id", table_name="comfort_pins")
|
||||
op.drop_table("comfort_pins")
|
||||
@@ -0,0 +1,41 @@
|
||||
"""add download_jobs.error_detail — the domain-specific entry of the failure catalog (E11.5 S1)
|
||||
|
||||
Revision ID: 0084_dl_job_error_detail
|
||||
Revises: 0083_comfort_pins
|
||||
Create Date: 2026-09-13
|
||||
|
||||
`error_code` is the generic category (it decides whether a Retry is offered). `error_detail` names
|
||||
the host-specific catalog entry `downloads.errors.classify` matched on top of it — `facebook.login_wall`
|
||||
and its siblings — so the UI can say what went wrong ON THAT SITE and what to do about it, instead of
|
||||
the category's generic sentence. Nullable: most failures have no host-specific entry.
|
||||
|
||||
Backfill: a job's code is written once, at failure time, so a new rule never reaches the rows that
|
||||
failed before it. The one shape measured on prod (2026-09-13, jobs 9/66/75/145/156/157/182/256/257) is
|
||||
yt-dlp's `[facebook] <id>: Cannot parse data`, stored as `unknown`. Without a cookie that is Facebook's
|
||||
login wall. Whether a cookie was attached at the time is not recorded anywhere, so every old row gets
|
||||
the no-cookie entry — on prod no account had a Facebook cookie, so that is what they all were.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0084_dl_job_error_detail"
|
||||
down_revision: str | None = "0083_comfort_pins"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("download_jobs", sa.Column("error_detail", sa.String(length=48), nullable=True))
|
||||
op.execute(
|
||||
sa.text(
|
||||
"UPDATE download_jobs SET error_code = 'login_required', "
|
||||
"error_detail = 'facebook.login_wall' "
|
||||
"WHERE (error_code IS NULL OR error_code = 'unknown') "
|
||||
"AND error ILIKE '%[facebook]%cannot parse data%'"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("download_jobs", "error_detail")
|
||||
@@ -0,0 +1,26 @@
|
||||
"""add users.last_login_at — when the account last signed in (E22 S1)
|
||||
|
||||
Revision ID: 0085_users_last_login_at
|
||||
Revises: 0084_dl_job_error_detail
|
||||
Create Date: 2026-09-16
|
||||
|
||||
Stamped by the one sign-in path (`routes/auth._establish_session`), which the Google callback and
|
||||
password login share. No backfill is possible: the signal exists only at sign-in, so existing rows
|
||||
read NULL until each account signs in again, and the admin UI says so rather than inventing a date.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0085_users_last_login_at"
|
||||
down_revision: str | None = "0084_dl_job_error_detail"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("users", sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "last_login_at")
|
||||
@@ -0,0 +1,48 @@
|
||||
"""add log_events — stored log events for diagnostics (E22 S3)
|
||||
|
||||
Revision ID: 0086_log_events
|
||||
Revises: 0085_users_last_login_at
|
||||
Create Date: 2026-09-17
|
||||
|
||||
Written by app/logsink.py's background writer (WARNING and above, request lines), pruned by the log_gc
|
||||
job after `log_retention_days`. Sized against prod on 2026-09-16: ~2k rows on a normal day, ~14k on a
|
||||
YouTube-RSS-outage day before flood control — a few MB over the 7-day window. `user_id` carries no
|
||||
foreign key on purpose (see models.LogEvent).
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0086_log_events"
|
||||
down_revision: str | None = "0085_users_last_login_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"log_events",
|
||||
sa.Column("id", sa.BigInteger(), primary_key=True),
|
||||
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("level", sa.String(10), nullable=False),
|
||||
sa.Column("sev", sa.SmallInteger(), nullable=False),
|
||||
sa.Column("source", sa.String(16), nullable=False),
|
||||
sa.Column("logger", sa.String(128), nullable=False),
|
||||
sa.Column("event", sa.String(64), nullable=True),
|
||||
sa.Column("msg", sa.Text(), nullable=False),
|
||||
sa.Column("attrs", postgresql.JSONB(), nullable=True),
|
||||
sa.Column("exc", sa.Text(), nullable=True),
|
||||
sa.Column("request_id", sa.String(64), nullable=True),
|
||||
sa.Column("user_id", sa.Integer(), nullable=True),
|
||||
sa.Column("debug_session_id", sa.String(64), nullable=True),
|
||||
sa.Column("job_id", sa.Integer(), nullable=True),
|
||||
sa.Column("version", sa.String(64), nullable=True),
|
||||
)
|
||||
op.create_index("ix_log_events_ts", "log_events", ["ts"])
|
||||
op.create_index("ix_log_events_request_id", "log_events", ["request_id"])
|
||||
op.create_index("ix_log_events_user_ts", "log_events", ["user_id", "ts"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("log_events")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""add users.diagnostics_level + diagnostics_until — a user's consented diagnostics level (E22 S5)
|
||||
|
||||
Revision ID: 0087_users_diagnostics_level
|
||||
Revises: 0086_log_events
|
||||
Create Date: 2026-09-17
|
||||
|
||||
Both NULL = the base level (Alap). A raised level is ALWAYS paired with its expiry, computed by the
|
||||
server at consent time (`app/diagnostics.py`), so no client can ask for a level that never ends.
|
||||
Columns rather than `users.preferences`: that JSON is written wholesale by the client, and a consent
|
||||
with a lifetime must not be something a client can write.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0087_users_diagnostics_level"
|
||||
down_revision: str | None = "0086_log_events"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("users", sa.Column("diagnostics_level", sa.String(8), nullable=True))
|
||||
op.add_column(
|
||||
"users", sa.Column("diagnostics_until", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("users", "diagnostics_until")
|
||||
op.drop_column("users", "diagnostics_level")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""add channels.feed_fallback_at — when a channel's uploads were last read through the API because
|
||||
its RSS feed would not answer
|
||||
|
||||
Revision ID: 0088_channels_feed_fallback_at
|
||||
Revises: 0087_users_diagnostics_level
|
||||
Create Date: 2026-09-19
|
||||
|
||||
YouTube's RSS feeds go dark for hours at a time, per channel (2026-09-18/19: 393 of 396 answered 404
|
||||
for four hours; the same channel answers 404 to every retry). The RSS poll then reads the newest
|
||||
uploads through the Data API instead, at a quota cost, and at most once per `rss_fallback_hours` per
|
||||
channel: this column is that clock. NULL = never needed it.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0088_channels_feed_fallback_at"
|
||||
down_revision: str | None = "0087_users_diagnostics_level"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"channels", sa.Column("feed_fallback_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("channels", "feed_fallback_at")
|
||||
@@ -0,0 +1,81 @@
|
||||
"""add debug_sessions + debug_screenshots, and index log_events.debug_session_id (E16)
|
||||
|
||||
Revision ID: 0089_debug_sessions
|
||||
Revises: 0088_channels_feed_fallback_at
|
||||
Create Date: 2026-09-19
|
||||
|
||||
A debug session is one recording: Start → reproduce → Stop (app/debugsession.py). Every log line
|
||||
of that user's requests while it runs carries its id in `log_events.debug_session_id` — the column
|
||||
E22 created and left NULL — and the issue it ends in is built from exactly those rows, so the column
|
||||
gets its index now. Partial: nearly every row has no session, and an index of NULLs would be most
|
||||
of the table for nothing.
|
||||
|
||||
At most one OPEN session per user, held by a partial unique index rather than by the start handler
|
||||
alone: two tabs pressing Start at once would otherwise both pass a "none open" check.
|
||||
|
||||
Screenshots are rows, not files: bounded (size and count per session, app/debugsession.py), gone
|
||||
with their session by cascade, and a self-host has no file volume to configure for them.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0089_debug_sessions"
|
||||
down_revision: str | None = "0088_channels_feed_fallback_at"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"debug_sessions",
|
||||
sa.Column("id", sa.String(16), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer,
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("level", sa.String(8), nullable=False),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("until", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("ended_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("ended_reason", sa.String(16), nullable=True),
|
||||
sa.Column("issue_url", sa.String(512), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
"uq_debug_sessions_open_per_user",
|
||||
"debug_sessions",
|
||||
["user_id"],
|
||||
unique=True,
|
||||
postgresql_where=sa.text("ended_at IS NULL"),
|
||||
)
|
||||
op.create_table(
|
||||
"debug_screenshots",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column(
|
||||
"session_id",
|
||||
sa.String(16),
|
||||
sa.ForeignKey("debug_sessions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("mime", sa.String(16), nullable=False),
|
||||
sa.Column("size", sa.Integer, nullable=False),
|
||||
sa.Column("data", sa.LargeBinary, nullable=False),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_log_events_debug_session",
|
||||
"log_events",
|
||||
["debug_session_id"],
|
||||
postgresql_where=sa.text("debug_session_id IS NOT NULL"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_log_events_debug_session", table_name="log_events")
|
||||
op.drop_table("debug_screenshots")
|
||||
op.drop_index("uq_debug_sessions_open_per_user", table_name="debug_sessions")
|
||||
op.drop_table("debug_sessions")
|
||||
@@ -0,0 +1,27 @@
|
||||
"""add debug_sessions.draft — the issue being written from a session (E16 S3)
|
||||
|
||||
Revision ID: 0090_debug_session_draft
|
||||
Revises: 0089_debug_sessions
|
||||
Create Date: 2026-09-19
|
||||
|
||||
The composer saves as it goes: the title, the rendered markdown body that an export or an issue
|
||||
carries, the log rows the user took out, and the form's own fields so reopening it later (Settings →
|
||||
Diagnosztika lists past sessions) shows what was written. NULL until the composer first saves.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
revision: str = "0090_debug_session_draft"
|
||||
down_revision: str | None = "0089_debug_sessions"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("debug_sessions", sa.Column("draft", JSONB, nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("debug_sessions", "draft")
|
||||
@@ -13,16 +13,18 @@ Three layers, by *who* runs the code — not by feature:
|
||||
response models, status-code mapping, serialization. It owns logic that runs *only* on a request.
|
||||
Business logic that a request happens to trigger does **not** live here if anything else runs it too.
|
||||
|
||||
2. **`plex/`, `downloads/`, `sync/` — domain logic shared by more than one caller.** Anything the
|
||||
2. **`plex/`, `downloads/`, `sync/`, `issues/` — domain logic shared by more than one caller.** Anything the
|
||||
**scheduler or the worker** runs (not just a request) lives in one of these domain packages, so the
|
||||
route, the worker, and the scheduler share one definition. Examples from this codebase:
|
||||
`downloads/service.py` owns enqueue + the job state transitions (routes call it); `plex/sync.py` is
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline.
|
||||
run by both the admin route and the scheduler; `downloads/pipeline.py` is the worker's child pipeline;
|
||||
`issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker.
|
||||
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, … are single-file primitives many
|
||||
layers import. They depend on nothing above them (a core module must never import a `routes/…`
|
||||
module).
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||
level) and `debugsession` (a recording that tags its user's lines, E16), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
module must never import a `routes/…` module).
|
||||
|
||||
**Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a
|
||||
*second* caller needs it (a worker as well as a route; a scheduler as well as a route), extract it down
|
||||
|
||||
@@ -27,9 +27,18 @@ class AuditAction:
|
||||
USER_SUSPEND = "user_suspend"
|
||||
USER_UNSUSPEND = "user_unsuspend"
|
||||
USER_DELETE = "user_delete"
|
||||
# E9.5 S5: an admin let a stuck registrant in, or re-sent their confirmation link. Both are
|
||||
# recorded because both are the admin acting ON someone's access, not merely reading a list.
|
||||
USER_ACTIVATE = "user_activate"
|
||||
USER_VERIFY_RESEND = "user_verify_resend"
|
||||
USER_SETPW_LINK = "user_setpw_link"
|
||||
INVITE_APPROVE = "invite_approve"
|
||||
INVITE_DENY = "invite_deny"
|
||||
INVITE_ADD = "invite_add"
|
||||
INVITE_DELETE = "invite_delete"
|
||||
# E9.5 S6: the admin chose to tell someone their address is whitelisted. Adding the address is
|
||||
# silent, so this is the act that actually reaches a person — and the one worth a record.
|
||||
INVITE_SEND = "invite_send"
|
||||
DEMO_ADD = "demo_add"
|
||||
DEMO_REMOVE = "demo_remove"
|
||||
DEMO_RESET = "demo_reset"
|
||||
@@ -44,8 +53,14 @@ class AuditAction:
|
||||
SYNC_RESUME = "sync_resume"
|
||||
# Scheduler job outcome / run-summary (actor_id set = a manual "run now"; NULL = automatic)
|
||||
JOB_RUN = "job_run"
|
||||
# Per-user download cookie/auth credential set or removed (the value is NEVER recorded)
|
||||
DOWNLOAD_CREDENTIAL_SET = "download_credential_set"
|
||||
DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete"
|
||||
# The log itself was cleared (a single row survives the clear as tamper-evidence)
|
||||
AUDIT_CLEAR = "audit_clear"
|
||||
# E16: a user sent a debug session's issue to the tracker — the one act that takes their log
|
||||
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
|
||||
ISSUE_CREATE = "issue_create"
|
||||
|
||||
|
||||
def record(
|
||||
|
||||
+234
-38
@@ -1,17 +1,20 @@
|
||||
import logging
|
||||
|
||||
import httpx
|
||||
from fastapi import BackgroundTasks, Depends, HTTPException, Request
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
from starlette.requests import HTTPConnection
|
||||
|
||||
from app import debugsession
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
from app.models import Invite, LogEvent, User
|
||||
from app.security import decrypt_optional
|
||||
from app.utils import canonical_email
|
||||
from app.refusal import refused
|
||||
|
||||
PASSWORD_MIN_LEN = 10
|
||||
|
||||
@@ -35,7 +38,7 @@ READ_SCOPES = f"{BASE_SCOPES} {READ_SCOPE}"
|
||||
WRITE_SCOPES = f"{BASE_SCOPES} {READ_SCOPE} {WRITE_SCOPE}"
|
||||
_YOUTUBE_SCOPES = frozenset({READ_SCOPE, WRITE_SCOPE})
|
||||
|
||||
log = logging.getLogger("siftlode.auth")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def google_enabled(db: Session) -> bool:
|
||||
@@ -63,16 +66,126 @@ def has_write_scope(user: User) -> bool:
|
||||
return WRITE_SCOPE in tok.scopes.split()
|
||||
|
||||
|
||||
def env_allowed_canonicals() -> set[str]:
|
||||
"""The env bootstrap allow-list, in CANONICAL form — the only shape it may be compared in.
|
||||
|
||||
S3 canonicalised the Invite branch of `is_allowed` and left this one matching the literal string,
|
||||
which is the 2026-09-03 lockout with a different source of truth: an operator who writes
|
||||
`first.last@gmail.com` into ALLOWED_EMAILS is refused when Google answers with the undotted
|
||||
primary, and a `+tag`ged entry can never match at all, because a Google account cannot contain a
|
||||
`+`. The addresses themselves stay literal wherever we MAIL them (`admin_notify_groups`); only
|
||||
the comparison is canonical.
|
||||
"""
|
||||
return env_admin_canonicals() | {canonical_email(e) for e in settings.allowed_email_set}
|
||||
|
||||
|
||||
def env_admin_canonicals() -> set[str]:
|
||||
"""The env ADMIN_EMAILS bootstrap list, canonical — for the role check only. Kept separate from
|
||||
`env_allowed_canonicals` because being listed as an admin also grants access, but being allowed
|
||||
does not make anyone an admin."""
|
||||
return {canonical_email(e) for e in settings.admin_email_set}
|
||||
|
||||
|
||||
def is_allowed(db: Session, email: str) -> bool:
|
||||
"""May this email sign in? An approved Invite is the source of truth; the env
|
||||
ALLOWED_EMAILS/ADMIN_EMAILS remain a bootstrap fallback (e.g. a fresh DB)."""
|
||||
ALLOWED_EMAILS/ADMIN_EMAILS remain a bootstrap fallback (e.g. a fresh DB).
|
||||
|
||||
Matched on the CANONICAL form (utils.canonical_email), not the literal string. Gmail delivers
|
||||
every dotted / `+tagged` spelling to one mailbox, so approving one spelling always meant approving
|
||||
that mailbox — matching literally only produced the 2026-09-03 lockout, where an approved user's
|
||||
Google sign-in became a silent second access request because OAuth returned the undotted primary.
|
||||
|
||||
One canonical form can match SEVERAL invite rows (prod carries approved pairs for one mailbox,
|
||||
the oldest from 2026-06-16). An approved row among them grants access: the API cannot flip an
|
||||
approved invite to denied (`_decide` only accepts a pending one), so approved-beside-denied means
|
||||
the admin approved that mailbox under a different spelling, not that they revoked it. Revoking is
|
||||
deleting the invite row or suspending the account — both still work, since a deleted row is no
|
||||
longer here to match and suspension is enforced at sign-in.
|
||||
"""
|
||||
if not email:
|
||||
return False
|
||||
email = email.lower()
|
||||
inv = db.execute(select(Invite).where(Invite.email == email)).scalar_one_or_none()
|
||||
if inv is not None:
|
||||
return inv.status == "approved"
|
||||
return email in settings.allowed_email_set or email in settings.admin_email_set
|
||||
canon = canonical_email(email)
|
||||
statuses = (
|
||||
db.execute(select(Invite.status).where(Invite.email_canonical == canon)).scalars().all()
|
||||
)
|
||||
if statuses:
|
||||
return "approved" in statuses
|
||||
return canon in env_allowed_canonicals()
|
||||
|
||||
|
||||
def can_activate(db: Session, user: User) -> bool:
|
||||
"""Is this account merely waiting for an admin to finish letting it in?
|
||||
|
||||
True when it is inactive but its address is ALREADY allowed — the shape roadmap UX-25 describes:
|
||||
the admin approved the address, the person signed up afterwards, and nothing connects the two
|
||||
(the Access-requests queue only offers Approve on a still-pending invite).
|
||||
|
||||
Gated on `is_allowed`, the same function the sign-in paths consult, because activating an address
|
||||
that is NOT allowed manufactures an inconsistent account: password sign-in only checks
|
||||
`is_active` and would let them in, while their Google sign-in would be turned away and silently
|
||||
reopened as a fresh access request. Letting in someone with no approved invite is the whitelist
|
||||
box's job — that action says so out loud.
|
||||
|
||||
Lives in core so the admin surface and the /api/me badge count answer from ONE definition; a
|
||||
button that appears and a number that counts must never disagree about who is stuck.
|
||||
"""
|
||||
if user.is_demo or user.is_suspended or user.is_active:
|
||||
return False
|
||||
return is_allowed(db, user.email)
|
||||
|
||||
|
||||
def count_pending_activations(db: Session) -> int:
|
||||
"""How many sign-ups are waiting on an admin click (see can_activate). Drives the admin badge.
|
||||
|
||||
Deliberately NOT "everyone who cannot sign in": an account that has simply not confirmed its
|
||||
address yet is waiting on its owner, not on the admin, and badging that would nag about something
|
||||
the admin did not cause. It is still listed on the admin screen — with a re-send — because the
|
||||
mail may never have arrived.
|
||||
"""
|
||||
# Narrowed in SQL, then decided by can_activate itself — so the number the badge shows and the
|
||||
# button the admin sees cannot drift apart by one of them re-deriving the rule.
|
||||
rows = (
|
||||
db.execute(
|
||||
select(User).where(
|
||||
User.is_demo.is_(False),
|
||||
User.is_suspended.is_(False),
|
||||
User.is_active.is_(False),
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return sum(1 for u in rows if can_activate(db, u))
|
||||
|
||||
|
||||
def find_user_by_email(db: Session, email: str) -> User | None:
|
||||
"""The account someone means when they type an address at sign-in, password-reset or
|
||||
verify-resend. None when there isn't exactly one answer.
|
||||
|
||||
Exact spelling wins. Failing that, the address is resolved to its mailbox
|
||||
(utils.canonical_email) — so a password chosen at registration still works when the person types
|
||||
another spelling of their own Gmail, and so the three flows that look an account up by typed
|
||||
address stop being dead ends. Before this, requesting a reset for `xy@gmail.com` on an account
|
||||
registered as `x.y@gmail.com` issued nothing and said nothing: the endpoint answers uniformly by
|
||||
design, so the user got silence and no way in.
|
||||
|
||||
SEVERAL canonical candidates return None rather than a guess. That case is real — prod carries two
|
||||
accounts for one mailbox — and picking one of them for a password sign-in would hand someone the
|
||||
wrong account. The callers all treat None as "no such account", which is the uniform,
|
||||
non-enumerating answer they already gave.
|
||||
"""
|
||||
if not email:
|
||||
return None
|
||||
email = email.strip().lower()
|
||||
exact = db.execute(select(User).where(User.email == email)).scalar_one_or_none()
|
||||
if exact is not None:
|
||||
return exact
|
||||
candidates = (
|
||||
db.execute(select(User).where(User.email_canonical == canonical_email(email)))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return candidates[0] if len(candidates) == 1 else None
|
||||
|
||||
|
||||
def get_or_create_demo_user(db: Session) -> User:
|
||||
@@ -108,7 +221,7 @@ def revoke_google_token(token: str | None) -> None:
|
||||
timeout=10,
|
||||
)
|
||||
log.info("Google token revoke: status=%s", resp.status_code)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 — a revoke we cannot make must not fail the sign-out
|
||||
log.warning("Google token revoke failed", exc_info=True)
|
||||
|
||||
|
||||
@@ -119,6 +232,9 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
||||
touch any browser session — each caller handles its own."""
|
||||
email = user.email.lower()
|
||||
user_id = user.id
|
||||
# Read the language off the row while it still exists — the confirmation below is queued as a
|
||||
# background task that runs after this session (and this user) are gone.
|
||||
lang = mailtext.lang_for_user(user)
|
||||
tok = user.token
|
||||
# Both are stored encrypted — decrypt for the revoke call (the access-token fallback was passing
|
||||
# ciphertext, so a token row with no refresh token silently failed to revoke). decrypt_optional so a
|
||||
@@ -130,64 +246,115 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
||||
)
|
||||
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
|
||||
db.execute(delete(User).where(User.id == user_id))
|
||||
db.execute(delete(Invite).where(Invite.email == email))
|
||||
# Stored log events name the account by id with no foreign key to cascade from (models.LogEvent),
|
||||
# so they go explicitly — they are request history, i.e. personal data. Forgotten FIRST: this very
|
||||
# request's access line, a queued row, a later background task would otherwise put rows back
|
||||
# under the id after this DELETE (logs.forget_user).
|
||||
logs.forget_user(user_id)
|
||||
db.execute(delete(LogEvent).where(LogEvent.user_id == user_id))
|
||||
# Every canonically-equal invite goes, not just the literal spelling: is_allowed matches on the
|
||||
# canonical form, so leaving a sibling row behind would keep granting access to a mailbox whose
|
||||
# account was just erased — and prod really does carry approved pairs for one mailbox.
|
||||
#
|
||||
# Unless ANOTHER account still lives on that mailbox — prod carries that too (two user rows for
|
||||
# one Gmail mailbox). Then NO invite goes, not even the literal one: is_allowed matches
|
||||
# canonically, so every row here is what keeps the survivor allowed, and removing any of them
|
||||
# would refuse their next Google sign-in and reopen it as a fresh access request — this epic's
|
||||
# founding bug, caused by its own cleanup. `delete_invite` refuses for exactly this reason; the
|
||||
# user row is already gone by now, so this query sees only the others.
|
||||
canon = canonical_email(email)
|
||||
if db.execute(select(User.id).where(User.email_canonical == canon)).first() is None:
|
||||
db.execute(delete(Invite).where(Invite.email_canonical == canon))
|
||||
db.commit()
|
||||
if google_token:
|
||||
background.add_task(revoke_google_token, google_token)
|
||||
# Confirm the erasure to the (now former) user — GDPR good practice, and covers both the
|
||||
# self-service and admin deletion paths since both funnel through here.
|
||||
background.add_task(email_mod.send_account_deleted, email, operator_contact())
|
||||
background.add_task(email_mod.send_account_deleted, email, lang, operator_contact())
|
||||
|
||||
|
||||
ACTIVE_ACCOUNT_HEADER = "x-siftlode-account"
|
||||
|
||||
|
||||
def resolved_user_id(conn: HTTPConnection) -> tuple[int | None, bool]:
|
||||
def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int | None, bool]:
|
||||
"""Which account this connection acts as, and whether that's the session's default account.
|
||||
Takes any HTTPConnection — an HTTP Request OR a WebSocket — so the WS push channel shares this
|
||||
exact per-tab resolution instead of re-implementing it.
|
||||
|
||||
The signed cookie holds the *wallet* (`account_ids`, every account signed into this browser)
|
||||
plus a default `user_id`. A caller may override the default with the X-Siftlode-Account header,
|
||||
but ONLY for an account already in the wallet — so a tab can't impersonate an account that never
|
||||
authenticated here. Everything else (WebSocket, plain navigations) keeps using the cookie default.
|
||||
plus a default `user_id`. A caller may name its account with the X-Siftlode-Account header (or
|
||||
`?account=`), but ONLY an account in the wallet — so a tab can't impersonate an account that
|
||||
never authenticated here. A connection that names no account acts as the cookie default.
|
||||
|
||||
A connection naming an account NOT in the wallet acts as nobody (`None`) — typically a tab still
|
||||
pinned to an account that was signed out in another tab. It must not fall back to the default:
|
||||
that tab would go on acting as an account it never showed, its player checkpoints and pref saves
|
||||
landing there and its Sign out signing THAT account out. Only the bootstrap /api/me probe passes
|
||||
`strict=False` and gets the default, which is how such a tab learns who it now is (the frontend
|
||||
re-pins it and reloads). An account whose session ended while it stayed in the wallet (deleted,
|
||||
suspended, revoked) leaves the wallet when it is refused (`_session_user`), so a tab pinned to it
|
||||
takes the same way back.
|
||||
"""
|
||||
default_id = conn.session.get("user_id")
|
||||
wallet = conn.session.get("account_ids") or []
|
||||
# Header for normal XHR; ?account= for contexts that can't set headers (WebSocket, and a
|
||||
# plain <a> file download). Both are wallet-gated below, so neither can impersonate an
|
||||
# account that never signed into this browser.
|
||||
# plain <a> file download). Both are wallet-gated below.
|
||||
hdr = conn.headers.get(ACTIVE_ACCOUNT_HEADER) or conn.query_params.get("account")
|
||||
if hdr:
|
||||
try:
|
||||
hid = int(hdr)
|
||||
except ValueError:
|
||||
# Under `strict` this becomes a 401 and the tab authenticates as nobody, which reads
|
||||
# in the browser exactly like a session that expired. The value itself stays out —
|
||||
# it came from a request, and `wallet` says which accounts were actually on offer.
|
||||
log.debug(
|
||||
"the active-account header could not be read as an id",
|
||||
extra=logs.event("auth.account_header.invalid", accounts=len(wallet)),
|
||||
)
|
||||
hid = None
|
||||
if hid is not None and hid in wallet:
|
||||
return hid, hid == default_id
|
||||
if strict:
|
||||
return None, False
|
||||
return default_id, True
|
||||
|
||||
|
||||
def drop_from_wallet(session: dict, user_id: int) -> None:
|
||||
"""Take one account out of this browser's wallet (`account_ids`); the session default is untouched."""
|
||||
session["account_ids"] = [a for a in (session.get("account_ids") or []) if a != user_id]
|
||||
|
||||
|
||||
def _end_account_session(request: Request, user_id: int, is_default: bool) -> None:
|
||||
"""An account whose session here is no longer valid. Only nuke the whole session when it is the
|
||||
SESSION DEFAULT; a per-tab account just leaves the wallet (other tabs' default may still be valid).
|
||||
Leaving is what lets a tab pinned to it recover: its next /api/me names an account no longer signed
|
||||
in here and gets the default. Left in, it answered "signed out" — the landing page, with the default
|
||||
still signed in, and no way off it but closing the tab."""
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
else:
|
||||
drop_from_wallet(request.session, user_id)
|
||||
|
||||
|
||||
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
user_id, is_default = resolved_user_id(request)
|
||||
return _session_user(request, db, *resolved_user_id(request))
|
||||
|
||||
|
||||
def _session_user(request: Request, db: Session, user_id: int | None, is_default: bool) -> User:
|
||||
if not user_id:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
raise refused(401, "not_authenticated")
|
||||
user = db.get(User, user_id)
|
||||
if user is None or not user.is_active or user.is_suspended:
|
||||
# Suspended/deactivated mid-session → drop the session so the block takes effect at once.
|
||||
# But only nuke the whole session when the SESSION DEFAULT identity is gone; a stale
|
||||
# per-tab header account just 401s that one tab (other tabs' default may still be valid).
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
# Suspended/deactivated mid-session → end it so the block takes effect at once.
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise refused(401, "not_authenticated")
|
||||
# SA4: reject a cookie whose recorded epoch is behind the account's current one (revoked by a
|
||||
# password reset/change or "log out everywhere"). A cookie with no recorded epoch (pre-SA4, or an
|
||||
# account signed in before this shipped) is treated as epoch 0 — so the FIRST bump revokes it too.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(user_id), 0)) != (user.session_epoch or 0):
|
||||
if is_default:
|
||||
request.session.clear()
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
_end_account_session(request, user_id, is_default)
|
||||
raise refused(401, "not_authenticated")
|
||||
# Always keep the session default in the switchable list — covers sessions created before
|
||||
# multi-session existed (their account_ids was never seeded), so the switcher isn't empty.
|
||||
default_id = request.session.get("user_id")
|
||||
@@ -195,16 +362,44 @@ def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
if default_id and default_id not in accounts:
|
||||
accounts.append(default_id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
# Attribute this request's log lines to the account (E22). Here, not in the request middleware:
|
||||
# this is the one place a session becomes an account, and the middleware runs outside both the
|
||||
# session middleware and this task's context. `request.state` is shared across middleware layers
|
||||
# (it lives on the ASGI scope), so the access-log line can read it back. The user's consented
|
||||
# diagnostics level rides along: their requests are logged — and stored — in that detail (S5).
|
||||
# So does a debug session the request names (E16): only the caller's own open one counts, and a
|
||||
# header naming anything else is ignored rather than refused — a stale tab must keep working.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, until = debugsession.level_for(user, session)
|
||||
logs.set_user(user.id, level, until, session.id if session else None)
|
||||
request.state.user_id = user.id
|
||||
return user
|
||||
|
||||
|
||||
def optional_current_user(request: Request, db: Session = Depends(get_db)) -> User | None:
|
||||
"""Like `current_user` but returns None instead of raising 401 when there's no valid session.
|
||||
Used by the bootstrap /api/me probe so the logged-out landing doesn't log a failed request to
|
||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it)."""
|
||||
the browser console (a non-2xx fetch is a console error no matter how the app handles it).
|
||||
The one lenient resolution (`strict=False`): a tab naming an account no longer signed in here gets
|
||||
the default account, which is how it learns who it now is — see `resolved_user_id`. That includes an
|
||||
account refused right here, which `_session_user` has just taken out of the wallet."""
|
||||
user_id, is_default = resolved_user_id(request, strict=False)
|
||||
try:
|
||||
return current_user(request, db)
|
||||
return _session_user(request, db, user_id, is_default)
|
||||
# silent-ok: this is the anonymous case as much as the refused one — every landing page hit
|
||||
# comes through here. The second attempt below is the handling, and it reports.
|
||||
except HTTPException:
|
||||
if is_default:
|
||||
return None
|
||||
try:
|
||||
return _session_user(request, db, *resolved_user_id(request, strict=False))
|
||||
except HTTPException:
|
||||
# Both the pinned account AND the default one were refused, so the tab is signed out
|
||||
# without being told why — and it got here from a session that named a real account.
|
||||
log.info(
|
||||
"a session was resolved to nobody: neither the named nor the default account held",
|
||||
extra=logs.event("auth.session.probe_rejected", user_id=user_id),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -214,7 +409,11 @@ def require_human(user: User = Depends(current_user)) -> User:
|
||||
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
|
||||
where there's no scope check to lean on."""
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="Not available in the demo account.")
|
||||
# Structured, because `req` leaves 403 to the caller and `notifyActionError` only shows a
|
||||
# STRUCTURED detail — a plain string here reached the demo user as their caller's generic
|
||||
# "couldn't do that", with no hint that the demo account is the reason (round 6). Affordances
|
||||
# this gate sits behind are not all UI-gated on `is_demo` (the comfort shelf is not).
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
return user
|
||||
|
||||
|
||||
@@ -224,10 +423,7 @@ def require_write_scope(user: User = Depends(require_human)) -> User:
|
||||
checks that had drifted into three different 403 messages across channels/playlists. Layers on
|
||||
`require_human` because writing to YouTube needs a real, non-demo identity."""
|
||||
if not has_write_scope(user):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="Enable YouTube editing in Settings first.",
|
||||
)
|
||||
raise refused(403, "youtube_editing_off")
|
||||
return user
|
||||
|
||||
|
||||
@@ -235,7 +431,7 @@ def admin_user(user: User = Depends(current_user)) -> User:
|
||||
"""Dependency: require the signed-in user to be an admin. Single source of truth for the
|
||||
admin gate — every admin route/action depends on this rather than re-checking the role."""
|
||||
if user.role != "admin":
|
||||
raise HTTPException(status_code=403, detail="Admin only")
|
||||
raise refused(403, "admin_only")
|
||||
return user
|
||||
|
||||
|
||||
|
||||
+121
-7
@@ -1,6 +1,7 @@
|
||||
from pathlib import Path
|
||||
from typing import Literal, get_args, get_origin
|
||||
|
||||
from pydantic import model_validator
|
||||
from pydantic import ValidationInfo, field_validator, model_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Shipped placeholder; production must override it. Used by the startup guard below.
|
||||
@@ -15,11 +16,40 @@ def _version_from_file() -> str:
|
||||
v = p.read_text(encoding="utf-8").strip()
|
||||
if v:
|
||||
return v
|
||||
# silent-ok: this runs at import, and `app.logs` imports THIS module — a log call here
|
||||
# would be a circular import. The loop tries the other location, then falls back to "dev".
|
||||
except OSError:
|
||||
continue
|
||||
return "dev"
|
||||
|
||||
|
||||
# Closed value sets. A setting typed with one of these is a CHOICE: pydantic rejects anything else at
|
||||
# boot, and sysconfig derives the admin picker's options from the same Literal — so the list lives in
|
||||
# exactly one place and a consumer never has to guess what a typo should fall back to.
|
||||
SearchSource = Literal["scrape", "api"]
|
||||
DownloadLayout = Literal["plex", "flat"]
|
||||
LogLevel = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR"]
|
||||
LogFormat = Literal["json", "text"]
|
||||
TranscodeHw = Literal["auto", "on", "off"]
|
||||
# libx264's presets, fastest first (`placebo` left out on purpose: no measurable gain for its cost).
|
||||
TranscodePreset = Literal[
|
||||
"ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"
|
||||
]
|
||||
|
||||
|
||||
def choices_of(annotation: object) -> tuple[str, ...]:
|
||||
"""The allowed values of a choice-typed setting, or () for any other type."""
|
||||
return tuple(get_args(annotation)) if get_origin(annotation) is Literal else ()
|
||||
|
||||
|
||||
def canonical_choice(options: tuple[str, ...], raw: object) -> str | None:
|
||||
"""The option `raw` names, in its canonical spelling (any letter case, surrounding space
|
||||
ignored), or None. The ONE matching rule for a choice value from the env and from the admin
|
||||
API, so a spelling one accepts the other cannot refuse."""
|
||||
folded = str(raw).strip().casefold()
|
||||
return next((option for option in options if option.casefold() == folded), None)
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
|
||||
|
||||
@@ -57,7 +87,7 @@ class Settings(BaseSettings):
|
||||
# VPS Caddy front door. A request from any other peer (someone hitting the app port directly)
|
||||
# is NOT trusted and its raw socket IP is used, so it can't spoof its rate-limit identity via
|
||||
# XFF. Empty = trust no proxy (use the direct peer) — correct for a directly-exposed / local-dev
|
||||
# deploy. Comma-separated. See routes.auth._client_ip + the deploy docs.
|
||||
# deploy. Comma-separated. See ratelimit.client_key + the deploy docs.
|
||||
trusted_proxy_ips: str = ""
|
||||
|
||||
# --- Outbound email (onboarding: access-request + approval notices) ---
|
||||
@@ -85,10 +115,13 @@ class Settings(BaseSettings):
|
||||
search_daily_limit_per_user: int = 70
|
||||
# Source for live YouTube search: "scrape" (InnerTube, no API quota) or "api" (search.list,
|
||||
# 100 units/page). Scrape is the default; flip to "api" if scraping is blocked/breaks.
|
||||
search_source: str = "scrape"
|
||||
search_source: SearchSource = "scrape"
|
||||
# Recent-first backfill: how far back to fetch on the first pass per channel.
|
||||
backfill_recent_max_videos: int = 100
|
||||
backfill_recent_max_days: int = 365
|
||||
# A channel whose RSS feed has not answered for this many hours gets its newest uploads read
|
||||
# through the Data API instead (1 unit), at most once per this many hours. 0 = never.
|
||||
rss_fallback_hours: int = 2
|
||||
|
||||
# Shorts are confirmed by probing youtube.com/shorts/<id>. Only videos at or below
|
||||
# this duration (and not livestreams) are probed; longer videos are never Shorts.
|
||||
@@ -155,6 +188,9 @@ class Settings(BaseSettings):
|
||||
# per-client responses are flaky, and android_vr can get bot-flagged under heavy load) the
|
||||
# worker retries with the FALLBACK set — the POT-capable web clients (web needs the token +
|
||||
# Deno) plus android for audio. Both admin-tunable when YouTube shifts what works.
|
||||
# A SIGNED-IN download (the user added YouTube cookies) keeps only the cookie-capable clients
|
||||
# of these two sets — yt-dlp drops the rest, so android_vr / android / ios never run there.
|
||||
# See adapters.YouTubeAdapter._COOKIE_CAPABLE.
|
||||
download_player_clients: str = "android_vr"
|
||||
download_player_clients_fallback: str = "web_safari,web,android"
|
||||
|
||||
@@ -186,6 +222,21 @@ class Settings(BaseSettings):
|
||||
download_supervisor_poll_seconds: int = 3
|
||||
# Grace between SIGTERM and SIGKILL when stopping a child (matches run_ffmpeg's escalation).
|
||||
download_kill_grace_seconds: int = 5
|
||||
# --- worker-child resource de-prioritization ---
|
||||
# The download/edit child (yt-dlp + ffmpeg) shares ONE LAN box with the API, Postgres, and Plex
|
||||
# playback. Without a priority handicap a CPU re-encode pegs every core for ~30s and starves web
|
||||
# requests + playback until it finishes. The child lowers its OWN priority at startup — inherited
|
||||
# by every ffmpeg/yt-dlp descendant — so any foreground work always wins. Each knob is best-effort
|
||||
# and independently disable-able with 0.
|
||||
download_child_nice: int = 10 # CPU nice increment applied to the child; 0 = don't renice
|
||||
download_child_ionice_class: int = 3 # best-effort IO class (3 = idle); 0 = leave IO untouched
|
||||
# Cap the browser-safe re-encode's encoder threads so a transcode can't claim every core. 0 =
|
||||
# reserve one core for foreground work (max(1, cpu_count - 1)); >0 pins exactly that many.
|
||||
download_ffmpeg_threads: int = 0
|
||||
# Throttle the retention GC's whole-tree orphan file sweep so its rglob can't monopolize disk IO
|
||||
# in the API process: sleep `pause_ms` after every `batch` files touched. pause_ms=0 disables.
|
||||
download_gc_sweep_pause_ms: int = 20
|
||||
download_gc_sweep_batch: int = 400
|
||||
# Total cache size cap across ALL users (bytes). When >0 and exceeded, the GC evicts the
|
||||
# least-recently-accessed ready assets until under it. 0 = unlimited (disk is the limit).
|
||||
download_total_max_bytes: int = 0
|
||||
@@ -193,6 +244,10 @@ class Settings(BaseSettings):
|
||||
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
|
||||
download_default_max_concurrent: int = 2
|
||||
download_default_max_jobs: int = 50
|
||||
# Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker
|
||||
# finishes), an upload's size is known at ingest, so this bounds a single file directly; the
|
||||
# user's overall byte footprint is still governed by download_default_max_bytes.
|
||||
download_upload_max_bytes: int = 524_288_000 # 500 MiB
|
||||
# Retention: an asset expires this many days after its last access; the GC job then
|
||||
# deletes it (once no job references it) after an additional grace window.
|
||||
download_retention_days: int = 30
|
||||
@@ -202,10 +257,27 @@ class Settings(BaseSettings):
|
||||
audit_retention_days: int = 180
|
||||
audit_gc_minutes: int = 1440
|
||||
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
||||
download_layout: str = "plex"
|
||||
download_layout: DownloadLayout = "plex"
|
||||
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
||||
sponsorblock_default: bool = False
|
||||
|
||||
# --- Diagnostics / logging (E22) ---
|
||||
# BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at
|
||||
# startup and on change without a restart. This is what a process logs at before it can reach
|
||||
# the database — and the only knob the worker/runner children read, since they are short-lived.
|
||||
# TRACE (below DEBUG) is the "Teljes" diagnostics level; see app/logs.py.
|
||||
log_level: LogLevel = "INFO"
|
||||
# "json" (one object per line — what Loki and the log_events sink ingest) or "text" (a readable
|
||||
# line for a local terminal). Redaction applies to both.
|
||||
log_format: LogFormat = "json"
|
||||
# Also write events to the `log_events` table (app/logsink.py): WARNING and above, the request
|
||||
# lines, and — once they exist — client batches. Off only where no table is wanted (the tests).
|
||||
log_db_sink: bool = True
|
||||
# Days a `log_events` row is kept (the privacy policy promises this), and how often the log_gc job
|
||||
# enforces it — hourly, so a row never outlives the promise by more than an hour.
|
||||
log_retention_days: int = 7
|
||||
log_gc_minutes: int = 60
|
||||
|
||||
# --- Plex integration (optional) ---
|
||||
# The Plex module plays the LOCAL physical media file directly (our own ffmpeg/direct-serve);
|
||||
# Plex is used only for metadata + optional watch-state sync. Requires the Plex server to be
|
||||
@@ -232,14 +304,38 @@ class Settings(BaseSettings):
|
||||
plex_sync_interval_min: int = 360
|
||||
# How often the incremental Plex→Siftlode watch-state reconcile runs, in minutes (Phase C):
|
||||
# a cheap history + on-deck pull plus a re-push of any states that failed their immediate push.
|
||||
plex_watch_sync_interval_min: int = 30
|
||||
# 15 min (S2): with Plex now the source of truth (prompt write-back on the ~10s checkpoint +
|
||||
# Plex-authoritative pull), a faster incremental pull keeps Siftlode close to Plex-side changes.
|
||||
plex_watch_sync_interval_min: int = 15
|
||||
# How often the FULL watch-state reconcile runs, in minutes (Phase C). A whole-section rescan
|
||||
# that catches what the incremental history feed can't — notably an un-watch done on the Plex
|
||||
# side — so it runs far less often (daily) than the incremental pass.
|
||||
plex_watch_reconcile_interval_min: int = 1440
|
||||
# Max concurrent transcodes for the P3 fallback. Low by default — CPU-only transcode is
|
||||
# expensive; direct-serve (browser-compatible files) has no such limit.
|
||||
# Max concurrent transcode sessions (playable="transcode"). Low by default — the software-fallback
|
||||
# re-encode is CPU-heavy (the VAAPI hardware path is far cheaper and could sustain more); direct-serve
|
||||
# and remux (stream-copy) are not capped by this.
|
||||
plex_max_transcodes: int = 4
|
||||
# Max concurrent playback sessions of ANY kind — remux (stream-copy) included. Cheap on the CPU but
|
||||
# each one is a whole segmented copy of the film on the scratch volume, written faster than
|
||||
# realtime. One account holds at most two regardless (plex/stream.py `_MAX_SESSIONS_PER_USER`).
|
||||
plex_max_sessions: int = 8
|
||||
# --- Transcode (playable="transcode") re-encode tuning (E6 S1) ---
|
||||
# Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the
|
||||
# source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render
|
||||
# node exists (skip the decodability gate); "off" = always software. See plex/stream.py `_select_encode`.
|
||||
plex_transcode_hw: TranscodeHw = "auto"
|
||||
# Software-fallback libx264 tuning (IGNORED on the VAAPI path). `preset` trades CPU for size; `crf`
|
||||
# trades quality for size (lower = better + bigger). `max_height` caps the SOFTWARE transcode's output
|
||||
# height (0 = keep source) so a CPU re-encode of a tall source stays affordable — the GPU path always
|
||||
# encodes at source resolution.
|
||||
# Loudness normalisation of the downmixed stereo audio on the HLS path (EBU R128 `loudnorm`).
|
||||
# OFF by default: it makes a film ~11 dB louder and narrows its dynamic range, which is a taste
|
||||
# call, not a fix. Only affects files that go through remux/transcode — a direct-played file is
|
||||
# served untouched, but those are browser-native stereo anyway, not the 5.1 mixes this is for.
|
||||
plex_audio_loudnorm: bool = False
|
||||
plex_transcode_preset: TranscodePreset = "veryfast"
|
||||
plex_transcode_crf: int = 20
|
||||
plex_transcode_max_height: int = 1080
|
||||
# Where on-the-fly HLS remux segments are written (transient, reaped). Empty → a `.plex-hls`
|
||||
# dir under download_root. On localdev the download_root is a slow Windows bind-mount, so point
|
||||
# this at a fast container-local path (e.g. /var/tmp/plex-hls); prod's download_root is fast
|
||||
@@ -255,6 +351,13 @@ class Settings(BaseSettings):
|
||||
tmdb_api_base: str = "https://api.themoviedb.org/3"
|
||||
# Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default.
|
||||
tmdb_language: str = "en-US"
|
||||
|
||||
# --- Bug reports (E16): where a debug session's issue goes. All three empty = no tracker; the
|
||||
# composer still exports a zip. The repository must be PRIVATE — an issue carries the reporter's
|
||||
# log — and app/issues/gitea.py refuses a public one. Runtime knobs (sysconfig, admin UI). ---
|
||||
issues_gitea_url: str = ""
|
||||
issues_gitea_repo: str = ""
|
||||
issues_gitea_token: str = ""
|
||||
# How often the TMDB enrichment job fetches a batch of not-yet-enriched Plex titles, in minutes.
|
||||
tmdb_enrich_interval_min: int = 120
|
||||
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||
@@ -286,6 +389,17 @@ class Settings(BaseSettings):
|
||||
https OAUTH_REDIRECT_URL as the signal for 'real deployment' vs local dev."""
|
||||
return self.oauth_redirect_url.lower().startswith("https://")
|
||||
|
||||
@field_validator("*", mode="before")
|
||||
@classmethod
|
||||
def _canonical_choice(cls, value: object, info: ValidationInfo) -> object:
|
||||
"""A choice setting accepts its value in any letter case (`LOG_LEVEL=debug`) and stores the
|
||||
canonical spelling; an unknown value still fails the boot with pydantic's own message."""
|
||||
field = cls.model_fields.get(info.field_name or "")
|
||||
options = choices_of(field.annotation) if field else ()
|
||||
if options and isinstance(value, str):
|
||||
return canonical_choice(options, value) or value
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _enforce_production_secrets(self) -> "Settings":
|
||||
"""Fail fast rather than boot a public instance with the shipped placeholder
|
||||
|
||||
@@ -20,6 +20,10 @@ engine = create_engine(
|
||||
max_overflow=30,
|
||||
pool_timeout=30,
|
||||
future=True,
|
||||
# A database error renders its bound parameters into its message — a title, an uploader, a
|
||||
# file path, a URL — and that message is logged and stored as a job's error text. The SQL
|
||||
# stays; the values are data, and the ids an operator needs are on the line already.
|
||||
hide_parameters=True,
|
||||
)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
|
||||
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, and the evidence a bug report is built from.
|
||||
|
||||
While a session is open its user's browser names it in `X-Debug-Session` on every request. The auth
|
||||
dependency accepts the header only for the caller's OWN open session (anything else is ignored, never
|
||||
an error — a stale tab must not break the app) and then logs that request at the session's level with
|
||||
every line tagged by its id; the browser tags its own events the same way. The issue is later built
|
||||
from exactly the rows carrying the tag, so what the review list shows is what gets sent.
|
||||
|
||||
Starting a session IS the consent for its level, for its duration: the start dialog carries the
|
||||
consent text. It needs no change to the user's own diagnostics level, and none is made — when the
|
||||
session ends the level in force is simply the user's own again.
|
||||
|
||||
A session is capped at `MAX_MINUTES`, set by the server at Start. An open one past its cap is over
|
||||
whether or not anything has closed it yet: it reads as over at once, a new Start closes it, and the
|
||||
hourly log job sweeps the rest. Sessions go with the logs they point at, after `log_retention_days`.
|
||||
|
||||
Core layer: imports `models`, `diagnostics`.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import StrEnum
|
||||
from typing import Literal, get_args
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import and_, delete, func, not_, select, update
|
||||
from sqlalchemy.orm import Session, defer
|
||||
|
||||
from app import diagnostics
|
||||
from app.models import DebugScreenshot, DebugSession, LogEvent, User
|
||||
|
||||
HEADER = "x-debug-session"
|
||||
MAX_MINUTES = diagnostics.SESSION_MINUTES
|
||||
SCREENSHOT_MAX_BYTES = 10 * 1024 * 1024
|
||||
SCREENSHOTS_MAX = 8
|
||||
# What one user may keep in screenshots across ALL their sessions (until retention takes them): the
|
||||
# per-session cap alone would let Start → 8 images → Stop repeat without end.
|
||||
SCREENSHOT_USER_MAX_BYTES = 100 * 1024 * 1024
|
||||
# The screenshot types kept, and the extension each is exported and sent under — one table, so an
|
||||
# accepted type always has a name.
|
||||
IMAGE_TYPES = {"image/png": "png", "image/jpeg": "jpg", "image/webp": "webp"}
|
||||
# What a screenshot is called in the zip and on the issue. `issues/bundle.py` builds the names from
|
||||
# this and `IMAGE_TYPES`; this module and `issues/gitea.py` read them back to tell a screenshot's
|
||||
# link from a log's. One definition, because three copies of a name is how the next rename breaks
|
||||
# `images_pending` into silence.
|
||||
SCREENSHOT_PREFIX = "screenshot-"
|
||||
# Rows an issue can carry. A 60-minute session at Teljes is a few thousand rows; past this the
|
||||
# bundle says how many it left out rather than growing without bound.
|
||||
BUNDLE_MAX = 5_000
|
||||
|
||||
SessionLevel = Literal["DEBUG", "TRACE"]
|
||||
SESSION_LEVELS: tuple[str, ...] = get_args(SessionLevel)
|
||||
# Detail order of the diagnostics levels: a session raises the level in force, never lowers it.
|
||||
_DETAIL = {"INFO": 0, "DEBUG": 1, "TRACE": 2}
|
||||
_ID_LEN = 16
|
||||
|
||||
|
||||
class EndReason(StrEnum):
|
||||
STOPPED = "stopped"
|
||||
EXPIRED = "expired"
|
||||
|
||||
|
||||
def is_live(session: DebugSession, now: datetime | None = None) -> bool:
|
||||
"""Open and within its cap — THE liveness rule: an open session past its cap is over."""
|
||||
return session.ended_at is None and session.until > (now or datetime.now(timezone.utc))
|
||||
|
||||
|
||||
def open_session(db: Session, user: User) -> DebugSession | None:
|
||||
"""The user's live session, or None. Read-only: one past its cap is over and reads as None;
|
||||
`start` and the hourly sweep are what close it."""
|
||||
session = _unclosed(db, user)
|
||||
if session is None or not is_live(session, datetime.now(timezone.utc)):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def _unclosed(db: Session, user: User) -> DebugSession | None:
|
||||
return db.scalar(
|
||||
select(DebugSession).where(DebugSession.user_id == user.id, DebugSession.ended_at.is_(None))
|
||||
)
|
||||
|
||||
|
||||
def start(db: Session, user: User, level: SessionLevel) -> DebugSession:
|
||||
"""Open a session, or return the live one — a second tab pressing Start joins the recording
|
||||
rather than ending it. One past its cap is closed first: the one-open-per-user index would
|
||||
refuse the new row beside it. Flushes; the caller commits. Two concurrent Starts can still both
|
||||
get here: the index refuses the second, and the caller joins the one that won."""
|
||||
now = datetime.now(timezone.utc)
|
||||
current = _unclosed(db, user)
|
||||
if current is not None:
|
||||
if is_live(current, now):
|
||||
return current
|
||||
end(current, EndReason.EXPIRED)
|
||||
db.flush()
|
||||
session = DebugSession(
|
||||
id=secrets.token_hex(_ID_LEN // 2),
|
||||
user_id=user.id,
|
||||
level=level,
|
||||
started_at=now,
|
||||
until=now + timedelta(minutes=MAX_MINUTES),
|
||||
)
|
||||
db.add(session)
|
||||
db.flush()
|
||||
return session
|
||||
|
||||
|
||||
def stop(session: DebugSession) -> None:
|
||||
"""The user's Stop. Past its cap the session had already ended on its own, and a late Stop must
|
||||
not rename that."""
|
||||
end(session, EndReason.STOPPED if is_live(session) else EndReason.EXPIRED)
|
||||
|
||||
|
||||
def end(session: DebugSession, reason: EndReason) -> None:
|
||||
"""Close a session. An expired one is dated at its cap, not at whenever it was noticed."""
|
||||
if session.ended_at is not None:
|
||||
return
|
||||
session.ended_at = (
|
||||
session.until
|
||||
if reason is EndReason.EXPIRED
|
||||
else min(datetime.now(timezone.utc), session.until)
|
||||
)
|
||||
session.ended_reason = reason
|
||||
|
||||
|
||||
def from_header(db: Session, user: User, value: str | None) -> DebugSession | None:
|
||||
"""The session a request names, if it is the caller's own and still live; else None."""
|
||||
if not value or len(value) != _ID_LEN or not all(c in "0123456789abcdef" for c in value):
|
||||
return None
|
||||
session = db.get(DebugSession, value)
|
||||
if (
|
||||
session is None
|
||||
or session.user_id != user.id
|
||||
or not is_live(session, datetime.now(timezone.utc))
|
||||
):
|
||||
return None
|
||||
return session
|
||||
|
||||
|
||||
def level_for(user: User, session: DebugSession | None) -> tuple[str, datetime | None]:
|
||||
"""The diagnostics level (and its expiry) a request is logged at: the user's own, raised to the
|
||||
session's while one is named — whichever records more.
|
||||
|
||||
A named session also caps the expiry at its own: this pair is what the process tracks under the
|
||||
session (app/logs.py), and an expiry outliving the recording would hold the root logger down
|
||||
after it ended."""
|
||||
own = diagnostics.effective_level(user)
|
||||
until = user.diagnostics_until if own != diagnostics.BASE else None
|
||||
if session is None:
|
||||
return own, until
|
||||
if _DETAIL[session.level] <= _DETAIL[own]:
|
||||
return own, min(until, session.until) if until else session.until
|
||||
return session.level, session.until
|
||||
|
||||
|
||||
def lock(db: Session, session: DebugSession) -> None:
|
||||
"""Re-read the session under a row lock, held until the caller commits. Everything that changes
|
||||
what an issue is made of — the draft, the screenshots, the send itself — takes it first: a send
|
||||
in flight sees none of them change under it, and each of them sees a send that committed."""
|
||||
db.refresh(session, with_for_update=True)
|
||||
|
||||
|
||||
def save_draft(session: DebugSession, draft: dict) -> None:
|
||||
"""Keep the composer's draft (the caller validated, locked and commits). Shape: `title`, `body`
|
||||
(the rendered markdown an export or an issue carries), `excluded` (log row ids taken out of the
|
||||
evidence) and `form` (the composer's own fields, to reopen it as it was). A delivery's progress
|
||||
is not the composer's to replace, and is kept."""
|
||||
progress = (session.draft or {}).get("delivery")
|
||||
session.draft = {**draft, "delivery": progress} if progress else draft
|
||||
|
||||
|
||||
# --- a delivery's progress -----------------------------------------------------------------
|
||||
# Written by app/issues/gitea.py into the draft; read HERE, so the send's decision and what the
|
||||
# composer shows are one reading of it.
|
||||
def delivery(session: DebugSession) -> dict:
|
||||
return dict((session.draft or {}).get("delivery") or {})
|
||||
|
||||
|
||||
def not_uploaded(session: DebugSession) -> list[str]:
|
||||
"""The files a send that stopped half-way has still to upload — by name, as the issue carries
|
||||
them. What the composer names to the user."""
|
||||
return list(delivery(session).get("pending") or [])
|
||||
|
||||
|
||||
def images_pending(session: DebugSession) -> bool:
|
||||
"""Whether the issue's body still has to be edited to show the screenshots it already carries
|
||||
(every upload went, the edit that shows them did not)."""
|
||||
state = delivery(session)
|
||||
shown = state.get("images_shown")
|
||||
return not shown and any(n.startswith(SCREENSHOT_PREFIX) for n in state.get("links") or {})
|
||||
|
||||
|
||||
def unfinished(session: DebugSession) -> bool:
|
||||
"""Whether a send owes the issue anything — the retry's question."""
|
||||
return bool(not_uploaded(session)) or images_pending(session)
|
||||
|
||||
|
||||
# A send claims the session while it talks to the tracker, instead of holding the row's lock across
|
||||
# those calls: a Stop, an autosave or a screenshot would queue behind a lock for the whole send.
|
||||
# They see the claim and are refused at once instead. A claim not renewed for this long was left by
|
||||
# a request that died mid-send (the process restarted), and the next send takes it over — which is
|
||||
# why a live send RENEWS it every time it reports progress (`set_delivery`). A claim is a lease, and
|
||||
# a send slower than the lease must not lose it while it is still running.
|
||||
SEND_CLAIM_SECONDS = 600
|
||||
|
||||
|
||||
def _claim(session: DebugSession) -> dict:
|
||||
claim = delivery(session).get("sending")
|
||||
return claim if isinstance(claim, dict) else {}
|
||||
|
||||
|
||||
def _stamp(token: str) -> dict:
|
||||
return {"at": datetime.now(timezone.utc).isoformat(), "by": token}
|
||||
|
||||
|
||||
def sending(session: DebugSession) -> bool:
|
||||
"""Whether a send is under way right now (a live claim)."""
|
||||
at = _claim(session).get("at")
|
||||
if not at:
|
||||
return False
|
||||
return (datetime.now(timezone.utc) - datetime.fromisoformat(at)).total_seconds() < (
|
||||
SEND_CLAIM_SECONDS
|
||||
)
|
||||
|
||||
|
||||
def claim_send(session: DebugSession) -> str:
|
||||
"""Mark the session as being sent, and return the token that OWNS this claim. The caller holds
|
||||
the lock and commits, which is what makes the claim visible to everyone else."""
|
||||
token = uuid4().hex
|
||||
_write_delivery(session, {**delivery(session), "sending": _stamp(token)})
|
||||
return token
|
||||
|
||||
|
||||
def release_send(session: DebugSession, token: str) -> None:
|
||||
"""Give back our OWN claim, and only ours: one that aged out and was taken over belongs to the
|
||||
send holding it now, and handing that one back would let a third send in beside it."""
|
||||
if _claim(session).get("by") != token:
|
||||
return
|
||||
_write_delivery(session, {k: v for k, v in delivery(session).items() if k != "sending"})
|
||||
|
||||
|
||||
class SendClaimLost(Exception):
|
||||
"""This send's claim is no longer on the row: it aged out and another send took it over. The
|
||||
send must stop — what it uploads from here it can no longer record, so a retry would send it
|
||||
again — and it must not write, because the row now holds the other send's progress."""
|
||||
|
||||
|
||||
def claim_owner(db: Session, session: DebugSession) -> str | None:
|
||||
"""Who holds the row's claim RIGHT NOW, read from the ROW and not from the object.
|
||||
|
||||
`expire_on_commit` is off (app/db.py), so a checkpoint does not re-read: a send that stalled
|
||||
past its lease still carries its own dead token in memory long after another send took over.
|
||||
A targeted scalar, not `lock`/`db.refresh` — a refresh here would discard the `issue_url` that
|
||||
`gitea.deliver` assigns one line before its first progress report, and that link is the one
|
||||
thing the send must not lose."""
|
||||
return db.scalar(
|
||||
select(DebugSession.draft["delivery"]["sending"]["by"].as_string()).where(
|
||||
DebugSession.id == session.id
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def set_delivery(db: Session, session: DebugSession, state: dict, token: str) -> None:
|
||||
"""Write a delivery's progress (app/issues/gitea.py keeps it; this module reads it), and renew
|
||||
the claim while doing it — a send still reporting progress is still alive, so one slower than
|
||||
the lease cannot expire while it is still running.
|
||||
|
||||
The claim is not the caller's to carry: `gitea.deliver` threads ONE snapshot of this dict
|
||||
through a whole send, so a progress report that carried the claim wrote a stale one back at
|
||||
every step. It is stamped from `token` here instead, and only after the ROW confirms the claim
|
||||
is still ours. Without that check the write is a full-column overwrite with no WHERE: a send
|
||||
that lost its lease came back and stomped the take-over send's committed links, then put its
|
||||
own dead token back on an idle session (deep-review round 4). `release_send` has always made
|
||||
the same check; this is the other half of it."""
|
||||
if claim_owner(db, session) != token:
|
||||
raise SendClaimLost(session.id)
|
||||
state = {k: v for k, v in state.items() if k != "sending"}
|
||||
state["sending"] = _stamp(token)
|
||||
_write_delivery(session, state)
|
||||
|
||||
|
||||
def _write_delivery(session: DebugSession, state: dict) -> None:
|
||||
# A new dict, so the JSONB column sees the change.
|
||||
session.draft = {**(session.draft or {}), "delivery": state}
|
||||
|
||||
|
||||
def serialize(session: DebugSession | None, screenshots: int = 0) -> dict | None:
|
||||
if session is None:
|
||||
return None
|
||||
return {
|
||||
"id": session.id,
|
||||
"level": session.level,
|
||||
"started_at": session.started_at.isoformat(),
|
||||
"until": session.until.isoformat(),
|
||||
"ended_at": session.ended_at.isoformat() if session.ended_at else None,
|
||||
"ended_reason": session.ended_reason,
|
||||
"issue_url": session.issue_url,
|
||||
# A send that stopped half-way: the files it has still to upload, and whether the issue's
|
||||
# body has still to be edited to show the screenshots it already carries. Both are true of a
|
||||
# send that is RUNNING too — every step commits its progress — so `sending` goes with them:
|
||||
# without it a reload or a second tab renders a live send as a failed one, and offers a
|
||||
# retry that is certain to be refused.
|
||||
"pending_files": not_uploaded(session),
|
||||
"images_pending": images_pending(session),
|
||||
"sending": sending(session),
|
||||
"screenshots": screenshots,
|
||||
"draft": session.draft,
|
||||
}
|
||||
|
||||
|
||||
def view(db: Session, session: DebugSession) -> dict:
|
||||
"""The session as the API serves it — one shape wherever it is served."""
|
||||
return serialize(session, screenshot_count(db, session)) or {}
|
||||
|
||||
|
||||
def screenshot_count(db: Session, session: DebugSession) -> int:
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.count())
|
||||
.select_from(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
# --- screenshots ---------------------------------------------------------------------------
|
||||
# By content, not by the declared type: what is stored is served back to a browser as an image.
|
||||
_SIGNATURES = (
|
||||
(b"\x89PNG\r\n\x1a\n", "image/png"),
|
||||
(b"\xff\xd8\xff", "image/jpeg"),
|
||||
)
|
||||
|
||||
|
||||
def sniff(data: bytes) -> str | None:
|
||||
"""The image type `data` really is — one of IMAGE_TYPES — or None."""
|
||||
for magic, mime in _SIGNATURES:
|
||||
if data.startswith(magic):
|
||||
return mime
|
||||
if len(data) >= 12 and data[:4] == b"RIFF" and data[8:12] == b"WEBP":
|
||||
return "image/webp"
|
||||
return None
|
||||
|
||||
|
||||
def lock_user(db: Session, user_id: int) -> None:
|
||||
"""Hold this user's own lock for the rest of the transaction. A session's row lock bounds what
|
||||
THAT session may keep; the byte cap below is the user's across all of them, and a lock on one
|
||||
session cannot serialize a check that reads them all — two uploads to two sessions both read the
|
||||
total before either commits, and both pass."""
|
||||
db.execute(select(func.pg_advisory_xact_lock(user_id)))
|
||||
|
||||
|
||||
def stored_screenshot_bytes(db: Session, user_id: int) -> int:
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.sum(DebugScreenshot.size), 0))
|
||||
.join(DebugSession, DebugSession.id == DebugScreenshot.session_id)
|
||||
.where(DebugSession.user_id == user_id)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
class ScreenshotRefused(Exception):
|
||||
"""A screenshot that cannot be kept; `code` is the refusal's machine word."""
|
||||
|
||||
def __init__(self, code: str) -> None:
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def add_screenshot(db: Session, session: DebugSession, data: bytes) -> DebugScreenshot:
|
||||
"""Keep one screenshot. Locks the user and the session, and flushes; the caller commits.
|
||||
Refused while a send is under way or once the issue was sent (it can no longer carry it), past
|
||||
the per-session count or the user's storage, or when it is not an image we can show."""
|
||||
if not data:
|
||||
raise ScreenshotRefused("empty")
|
||||
mime = sniff(data)
|
||||
if mime is None:
|
||||
raise ScreenshotRefused("not_an_image")
|
||||
# Under both locks: two uploads at once cannot both pass the count (the session's), nor both
|
||||
# pass the user's byte cap from two different sessions (the user's). Always in this order.
|
||||
lock_user(db, session.user_id)
|
||||
lock(db, session)
|
||||
if sending(session):
|
||||
raise ScreenshotRefused("sending")
|
||||
if session.issue_url is not None:
|
||||
raise ScreenshotRefused("already_sent")
|
||||
if screenshot_count(db, session) >= SCREENSHOTS_MAX:
|
||||
raise ScreenshotRefused("too_many")
|
||||
if stored_screenshot_bytes(db, session.user_id) + len(data) > SCREENSHOT_USER_MAX_BYTES:
|
||||
raise ScreenshotRefused("storage_full")
|
||||
shot = DebugScreenshot(
|
||||
session_id=session.id,
|
||||
created_at=datetime.now(timezone.utc),
|
||||
mime=mime,
|
||||
size=len(data),
|
||||
data=data,
|
||||
)
|
||||
db.add(shot)
|
||||
db.flush()
|
||||
return shot
|
||||
|
||||
|
||||
def remove_screenshot(db: Session, session: DebugSession, shot: DebugScreenshot) -> None:
|
||||
"""Delete one screenshot, under the lock. Refused while a send is under way or unfinished: a
|
||||
delivery names the screenshots by their order, and removing one would renumber what it still
|
||||
has to send. Once the issue has everything, this only drops OUR copy — the issue keeps its
|
||||
own. The caller commits."""
|
||||
lock(db, session)
|
||||
if sending(session):
|
||||
raise ScreenshotRefused("sending")
|
||||
if unfinished(session):
|
||||
raise ScreenshotRefused("already_sent")
|
||||
db.delete(shot)
|
||||
db.flush()
|
||||
|
||||
|
||||
def screenshots(db: Session, session: DebugSession) -> list[DebugScreenshot]:
|
||||
"""The session's screenshots, oldest first, WITHOUT their bytes loaded."""
|
||||
return list(
|
||||
db.scalars(
|
||||
select(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
.options(defer(DebugScreenshot.data))
|
||||
.order_by(DebugScreenshot.id)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --- the evidence --------------------------------------------------------------------------
|
||||
# The browser's own log upload (`POST` here, every few seconds while recording) is how the evidence
|
||||
# arrives, not part of it: its server lines would fill the bundle with the recording's plumbing. The
|
||||
# browser leaves its copies out for the same reason (frontend lib/diagnostics `isOwnRequest`).
|
||||
INGEST_PATH = "/api/diagnostics/events"
|
||||
|
||||
|
||||
def _evidence(session: DebugSession):
|
||||
upload_line = and_(
|
||||
LogEvent.event == "http.request", LogEvent.attrs["path"].astext == INGEST_PATH
|
||||
)
|
||||
# `IS TRUE`, not the bare condition: for a row with no event or no attrs it is NULL, and
|
||||
# `NOT NULL` would drop exactly the rows that are evidence.
|
||||
return (LogEvent.debug_session_id == session.id, not_(upload_line.is_(True)))
|
||||
|
||||
|
||||
def bundle(
|
||||
db: Session, session: DebugSession, *, drop_excluded: bool = False
|
||||
) -> tuple[list[LogEvent], int]:
|
||||
"""The session's log rows, browser and server interleaved in time, capped at BUNDLE_MAX — and
|
||||
how many rows it has in all, so a cut bundle can say so. `drop_excluded` leaves out the rows
|
||||
the user took out in the composer: what an export or an issue carries."""
|
||||
where = list(_evidence(session))
|
||||
excluded = (session.draft or {}).get("excluded") or [] if drop_excluded else []
|
||||
if excluded:
|
||||
where.append(LogEvent.id.not_in(excluded))
|
||||
rows = list(
|
||||
db.scalars(
|
||||
select(LogEvent).where(*where).order_by(LogEvent.ts, LogEvent.id).limit(BUNDLE_MAX)
|
||||
)
|
||||
)
|
||||
total = len(rows)
|
||||
if total == BUNDLE_MAX:
|
||||
total = db.scalar(select(func.count()).select_from(LogEvent).where(*where)) or total
|
||||
return rows, total
|
||||
|
||||
|
||||
# --- housekeeping (the hourly log job) -----------------------------------------------------
|
||||
def sweep(db: Session) -> int:
|
||||
"""Stage closing every open session past its cap; returns how many."""
|
||||
now = datetime.now(timezone.utc)
|
||||
result = db.execute(
|
||||
update(DebugSession)
|
||||
.where(DebugSession.ended_at.is_(None), DebugSession.until <= now)
|
||||
.values(ended_at=DebugSession.until, ended_reason=EndReason.EXPIRED.value)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Stage deleting sessions (and by cascade their screenshots) started before the log retention
|
||||
window — the rows they would bundle are gone by then too. Returns how many."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, retention_days))
|
||||
result = db.execute(delete(DebugSession).where(DebugSession.started_at < cutoff))
|
||||
return result.rowcount or 0
|
||||
@@ -0,0 +1,65 @@
|
||||
"""A user's diagnostics level (E22 S5): what they consented to, and for how long.
|
||||
|
||||
Three levels, named in the UI Alap / Részletes / Teljes:
|
||||
|
||||
* INFO — the base, for everyone, no consent. The browser keeps its log in memory; nothing extra
|
||||
is sent or stored.
|
||||
* DEBUG — consented. The server logs this user's requests in detail and stores every line of
|
||||
them, and the browser streams its own log to `POST /api/diagnostics/events`.
|
||||
* TRACE — consented. As DEBUG, plus what the user typed into a search.
|
||||
|
||||
A raised level always lapses after `ELEVATION_HOURS`: the expiry is computed HERE at consent time,
|
||||
never taken from the client, and an expired level reads as INFO without anything having to clear it.
|
||||
The shared demo account cannot raise its level — its log would mix every visitor's session.
|
||||
|
||||
Core layer: imports `models` and `sysconfig`.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Literal, get_args
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.models import User
|
||||
|
||||
DiagnosticsLevel = Literal["INFO", "DEBUG", "TRACE"]
|
||||
LEVELS: tuple[str, ...] = get_args(DiagnosticsLevel)
|
||||
BASE: DiagnosticsLevel = "INFO"
|
||||
ELEVATION_HOURS = 24
|
||||
# How long a debug session records at most (E16, app/debugsession.py). Here, beside the other
|
||||
# consent window, because the start dialog's consent text states it and reads it from `state`.
|
||||
SESSION_MINUTES = 60
|
||||
|
||||
|
||||
def effective_level(user: User, now: datetime | None = None) -> DiagnosticsLevel:
|
||||
"""The level in force for `user` right now: the consented one until it expires, else BASE."""
|
||||
level, until = user.diagnostics_level, user.diagnostics_until
|
||||
if level not in LEVELS or level == BASE or until is None:
|
||||
return BASE
|
||||
if until <= (now or datetime.now(timezone.utc)):
|
||||
return BASE
|
||||
return level # type: ignore[return-value] — narrowed by the LEVELS check above
|
||||
|
||||
|
||||
def state(db: Session, user: User) -> dict:
|
||||
"""The `/api/me` view of it: the level in force, when a raised one lapses, and how long stored
|
||||
log lines are kept — the consent text states that number, so it must be the instance's own."""
|
||||
level = effective_level(user)
|
||||
return {
|
||||
"level": level,
|
||||
"until": user.diagnostics_until.isoformat() if level != BASE else None,
|
||||
"retention_days": sysconfig.get_int(db, "log_retention_days"),
|
||||
"session_minutes": SESSION_MINUTES,
|
||||
}
|
||||
|
||||
|
||||
def set_level(user: User, level: DiagnosticsLevel) -> None:
|
||||
"""Record a consent (or its withdrawal). The caller commits. Raising the level again restarts
|
||||
the clock — each consent is for the next ELEVATION_HOURS, not an extension of an older one."""
|
||||
if level == BASE:
|
||||
user.diagnostics_level = None
|
||||
user.diagnostics_until = None
|
||||
return
|
||||
user.diagnostics_level = level
|
||||
user.diagnostics_until = datetime.now(timezone.utc) + timedelta(hours=ELEVATION_HOURS)
|
||||
@@ -0,0 +1,10 @@
|
||||
"""E24 — the read-model behind the Home surface ("Mit nézzek most?").
|
||||
|
||||
The anti-paralysis answer layer: one hero suggestion, a cross-source resume rail, and the user's own
|
||||
comfort shelf, all served from our own Postgres with no upstream call at serve time (the same rule
|
||||
the Plex Discover surface follows).
|
||||
|
||||
Layering, mirroring `app/plex/`: `budget` (the time bands) and `cards` (the unified row shape) are
|
||||
pure; `queries` holds the SQL; `rails`, `pins` and `hero` compose. Nothing here imports from
|
||||
`app.routes` — `routes/discover.py` is a thin HTTP shell over these.
|
||||
"""
|
||||
@@ -0,0 +1,62 @@
|
||||
"""The time budget — E24's replacement for "what mood are you in?".
|
||||
|
||||
`random-gondolatok` 15/D asked for a mood questionnaire before the recommendations. The same file's
|
||||
later BA-interview note (#23) says the mood half is not what makes this work: the friction is
|
||||
deciding at all. A mood→genre mapping would also be invented data — we store no mood signal and never
|
||||
will — whereas "how much time do you have" is a question the library can actually answer, from a
|
||||
column we already have.
|
||||
|
||||
So the surface asks about TIME, and the answer is deterministic: four bands, no model, no latency.
|
||||
The band also picks the source for free — ten minutes is a YouTube video, an evening is a movie —
|
||||
which is exactly the cross-source behaviour the Home surface exists to provide.
|
||||
|
||||
The bands overlap on purpose. "I have 30 minutes" should not be answered with a two-minute Short just
|
||||
because it fits; a lower bound is as much a part of the promise as the upper one.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
Budget = Literal["short", "medium", "long", "any"]
|
||||
|
||||
DEFAULT_BUDGET: Budget = "any"
|
||||
|
||||
# budget -> (min_seconds, max_seconds); None = unbounded on that side.
|
||||
BANDS: dict[str, tuple[int | None, int | None]] = {
|
||||
"short": (None, 12 * 60), # "10 perc" — a bit of slack over the round number the chip shows
|
||||
"medium": (8 * 60, 40 * 60), # "30 perc"
|
||||
"long": (40 * 60, None), # "Egy este"
|
||||
"any": (None, None), # "Mindegy"
|
||||
}
|
||||
|
||||
|
||||
def band(budget: str) -> tuple[int | None, int | None]:
|
||||
"""The (min, max) seconds for a budget. Unknown budgets fall back to the unbounded band rather
|
||||
than raising: the route validates the vocabulary with a Literal, so anything reaching here with a
|
||||
bad value is our own bug, and an unfiltered shelf is a better failure than a 500 on the landing
|
||||
page."""
|
||||
return BANDS.get(budget, BANDS["any"])
|
||||
|
||||
|
||||
def remaining_seconds(duration_seconds: int | None, position_seconds: int) -> int | None:
|
||||
"""What is actually LEFT to watch. The question is "how much time do you have", so a 90-minute
|
||||
film with 8 minutes to go belongs in the ten-minute band, not the evening one. Unknown duration
|
||||
stays unknown — see `fits`."""
|
||||
if duration_seconds is None:
|
||||
return None
|
||||
return max(0, duration_seconds - max(0, position_seconds))
|
||||
|
||||
|
||||
def fits(seconds: int | None, budget: str) -> bool:
|
||||
"""Does a runtime fit the budget?
|
||||
|
||||
An unknown duration fits only the unbounded band. We cannot promise an item finishes inside ten
|
||||
minutes when we do not know how long it is, and quietly including it would make the one control
|
||||
on this page dishonest — the failure mode the house rules call an un-honest state."""
|
||||
lo, hi = band(budget)
|
||||
if seconds is None:
|
||||
return lo is None and hi is None
|
||||
if lo is not None and seconds < lo:
|
||||
return False
|
||||
if hi is not None and seconds > hi:
|
||||
return False
|
||||
return True
|
||||
@@ -0,0 +1,141 @@
|
||||
"""The unified discover card: one row shape the Home surface can render without knowing which
|
||||
module it came from.
|
||||
|
||||
This is deliberately a THIN mapping, not a new serializer layer. The per-source projections already
|
||||
exist and are already the shapes the frontend's `VideoCard` and `PlexPosterCard` consume —
|
||||
`feedquery.serialize` for YouTube, `plex.cards.movie_card`/`show_card`/`leaf_card` for Plex — so each
|
||||
unified card carries the native payload under `video` / `plex` alongside the handful of fields the
|
||||
rail chrome itself needs (title, thumbnail, progress, runtime). The Home page therefore renders with
|
||||
the existing card components rather than a third one, and a field added to a feed row or a Plex card
|
||||
still gets added in exactly one place. CLAUDE.md's "not wanted" list names a base serializer for
|
||||
precisely this reason.
|
||||
|
||||
`aspect` is the one genuinely new field. A rail has to know whether a tile is a 16:9 still or a 2:3
|
||||
poster, and that is a property of the ARTWORK, not of the module: a Plex episode ships a landscape
|
||||
still exactly like a YouTube video does, while a movie and a show ship posters. Emitting it lets the
|
||||
rail layer stay domain-free instead of branching on `source`.
|
||||
|
||||
`subtitle` only ever carries DATA — a channel name, a show title, a year. Nothing here composes an
|
||||
English phrase: this repo is HU+EN with zero key drift, and a sentence assembled in the backend is a
|
||||
string no translation file can reach. Anything that needs wording (season counts, "N videó") is left
|
||||
to the client, which has both the native payload and the translations.
|
||||
"""
|
||||
|
||||
from app.discover.budget import remaining_seconds
|
||||
from app.models import Channel, Playlist
|
||||
|
||||
# Plex artwork is a poster for the things you browse and a landscape still for the things you play.
|
||||
_POSTER_KINDS = ("movie", "show")
|
||||
|
||||
|
||||
def _base(
|
||||
*,
|
||||
source: str, # "youtube" | "plex"
|
||||
kind: str, # "video" | "movie" | "show" | "episode" | "channel" | "playlist"
|
||||
id: str,
|
||||
title: str | None,
|
||||
aspect: str, # "wide" (16:9) | "poster" (2:3)
|
||||
subtitle: str | None = None,
|
||||
thumb: str | None = None,
|
||||
duration_seconds: int | None = None,
|
||||
position_seconds: int = 0,
|
||||
status: str = "new",
|
||||
) -> dict:
|
||||
return {
|
||||
"source": source,
|
||||
"kind": kind,
|
||||
"id": id,
|
||||
"title": title or "",
|
||||
"subtitle": subtitle,
|
||||
"thumb": thumb,
|
||||
"aspect": aspect,
|
||||
"duration_seconds": duration_seconds,
|
||||
"position_seconds": position_seconds,
|
||||
"remaining_seconds": remaining_seconds(duration_seconds, position_seconds),
|
||||
"status": status,
|
||||
# Native payloads — at most one is ever set. The client opens the player from these.
|
||||
"video": None,
|
||||
"plex": None,
|
||||
}
|
||||
|
||||
|
||||
def key(card: dict) -> str:
|
||||
"""The identity of a card ACROSS sources — `<source>:<id>`. A Plex rating_key and a YouTube video
|
||||
id could in principle collide as bare strings, and the reroll's exclusion list has to be able to
|
||||
say "not that one" about either."""
|
||||
return f"{card['source']}:{card['id']}"
|
||||
|
||||
|
||||
def from_video(row: dict) -> dict:
|
||||
"""`row` is a `feedquery.serialize` dict — the same object the feed grid renders."""
|
||||
card = _base(
|
||||
source="youtube",
|
||||
kind="video",
|
||||
id=row["id"],
|
||||
title=row["title"],
|
||||
aspect="wide",
|
||||
subtitle=row.get("channel_title"),
|
||||
thumb=row.get("thumbnail_url"),
|
||||
duration_seconds=row.get("duration_seconds"),
|
||||
position_seconds=row.get("position_seconds") or 0,
|
||||
status=row.get("status") or "new",
|
||||
)
|
||||
card["video"] = row
|
||||
return card
|
||||
|
||||
|
||||
def from_plex(row: dict) -> dict:
|
||||
"""`row` is a `plex.cards` dict (movie / show / leaf). Its own `type` is the kind."""
|
||||
kind = row.get("type") or "movie"
|
||||
if kind == "episode":
|
||||
subtitle = row.get("show_title")
|
||||
elif kind == "show":
|
||||
subtitle = None # season count lives in the native payload; the client words it
|
||||
else:
|
||||
subtitle = str(row["year"]) if row.get("year") else None
|
||||
|
||||
card = _base(
|
||||
source="plex",
|
||||
kind=kind,
|
||||
id=row["id"],
|
||||
title=row.get("title"),
|
||||
aspect="poster" if kind in _POSTER_KINDS else "wide",
|
||||
subtitle=subtitle,
|
||||
thumb=row.get("thumb"),
|
||||
duration_seconds=row.get("duration_seconds"),
|
||||
position_seconds=row.get("position_seconds") or 0,
|
||||
status=row.get("status") or "new",
|
||||
)
|
||||
card["plex"] = row
|
||||
return card
|
||||
|
||||
|
||||
def from_channel(ch: Channel, unwatched: int) -> dict:
|
||||
"""A pinned channel. It is not itself playable — the shelf resolves it to its newest unwatched
|
||||
video on click (see `pins._shelf_card`) — so the tile advertises what is waiting there instead of a
|
||||
runtime it does not have."""
|
||||
card = _base(
|
||||
source="youtube",
|
||||
kind="channel",
|
||||
id=ch.id,
|
||||
title=ch.title,
|
||||
aspect="wide",
|
||||
thumb=ch.thumbnail_url,
|
||||
)
|
||||
card["unwatched_count"] = unwatched
|
||||
return card
|
||||
|
||||
|
||||
def from_playlist(pl: Playlist, item_count: int, thumb: str | None) -> dict:
|
||||
"""A pinned playlist. `thumb` is its first item's thumbnail — a shelf of blank tiles would defeat
|
||||
the point of a surface you are supposed to be able to scan at a glance."""
|
||||
card = _base(
|
||||
source="youtube",
|
||||
kind="playlist",
|
||||
id=str(pl.id),
|
||||
title=pl.name,
|
||||
aspect="wide",
|
||||
thumb=thumb,
|
||||
)
|
||||
card["item_count"] = item_count
|
||||
return card
|
||||
@@ -0,0 +1,108 @@
|
||||
"""The hero: the single suggestion the Home surface leads with.
|
||||
|
||||
One rule, in one function, because the page's entire promise is that it answers rather than offers.
|
||||
Three tiers, tried in order:
|
||||
|
||||
1. **resume** — something already started whose REMAINING time fits the budget. Almost always the
|
||||
right answer: it is the thing the user themselves chose most recently, and finishing it costs no
|
||||
decision at all.
|
||||
2. **comfort** — a pinned shelf entry that fits. The user's own declared fallback, which beats
|
||||
anything we could infer.
|
||||
3. **surprise** — a seeded draw from what is unseen and fits.
|
||||
|
||||
That order is the PAGE's answer. "Mást kérek" rotates it (`start`), so each press offers a different
|
||||
KIND of answer rather than the next row of the same list — the user's call, 2026-09-06, after the
|
||||
button was measured cycling nine resume cards and never once reaching the shelf.
|
||||
|
||||
The tier travels with the card so the client can label it honestly ("Ott hagytad abba" vs "A
|
||||
polcodról" vs "Meglepetés") instead of guessing from the shape of the payload — a kicker that says
|
||||
"where you left off" over a film the user has never opened is exactly the kind of quiet lie the house
|
||||
rules call a dishonest state.
|
||||
|
||||
Budgets are matched against REMAINING time throughout (`budget.remaining_seconds`, which equals the
|
||||
runtime for anything not yet started): with eight minutes left, a 90-minute film is a ten-minute
|
||||
answer.
|
||||
"""
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.discover import cards, pins, rails
|
||||
from app.discover.budget import fits
|
||||
from app.models import User
|
||||
|
||||
HERO_TIERS = ("resume", "comfort", "surprise")
|
||||
|
||||
# How deep the surprise tier draws before giving up. Deliberately deeper than one, so an excluded
|
||||
# pick moves the tier along instead of emptying it (the shuffle is seeded — the same seed always
|
||||
# hands back the same order, so without the headroom "already shown" would mean "nothing left").
|
||||
SURPRISE_DRAW = 12
|
||||
|
||||
|
||||
def pick(
|
||||
db: Session,
|
||||
user: User,
|
||||
budget: str,
|
||||
seed: int,
|
||||
*,
|
||||
resume: list[dict] | None = None,
|
||||
shelf: list[dict] | None = None,
|
||||
exclude: set[str] | None = None,
|
||||
start: int = 0,
|
||||
) -> dict | None:
|
||||
"""`{"tier": ..., "card": ...}`, or None when nothing at all fits.
|
||||
|
||||
`resume` and `shelf` may be passed pre-computed so the consolidated `/api/discover` response does
|
||||
not build the same two rails twice; the reroll endpoint passes neither and pays only for what it
|
||||
reaches.
|
||||
|
||||
`exclude` is what makes "Mást kérek" mean something. A new seed alone only re-draws the SURPRISE
|
||||
tier: the resume tier deterministically returns the most recently progressed thing that fits, so
|
||||
pressing reroll on a user who has something half-finished handed back the identical card and the
|
||||
button looked dead (measured in the browser, 2026-09-06). Skipping the cards already shown makes
|
||||
every press produce a different answer, at every tier.
|
||||
|
||||
`start` ROTATES which tier is tried first — 0 keeps the priority order (the page's own first
|
||||
answer), 1 leads with comfort, 2 with surprise. Exclusion alone was not enough: the resume rail
|
||||
holds up to 20 rows, so the button walked down it and never reached the shelf or a surprise at
|
||||
all (measured, 2026-09-06: twelve presses returned nine cards, every one of them `resume`). The
|
||||
fallthrough is kept, so a press does not come back empty just because the tier it led with has
|
||||
nothing to offer."""
|
||||
skip = exclude or set()
|
||||
|
||||
def _resume() -> dict | None:
|
||||
for card in rails.continue_watching(db, user) if resume is None else resume:
|
||||
playable = rails.playable(db, user.id, card)
|
||||
# Excluded on the card the USER saw (the show), not only on what it resolves to, so a
|
||||
# rerolled-past show does not come straight back as its own next episode.
|
||||
if cards.key(card) in skip or (playable is not None and cards.key(playable) in skip):
|
||||
continue
|
||||
if playable is not None and fits(playable["remaining_seconds"], budget):
|
||||
return playable
|
||||
return None
|
||||
|
||||
def _comfort() -> dict | None:
|
||||
if shelf is None:
|
||||
own = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
||||
else:
|
||||
own = shelf
|
||||
for card in pins.playables(own):
|
||||
if cards.key(card) not in skip and fits(card["remaining_seconds"], budget):
|
||||
return card
|
||||
return None
|
||||
|
||||
def _surprise() -> dict | None:
|
||||
# A wider draw than the one card we need: the shuffle is seeded, so without the headroom an
|
||||
# excluded pick would leave the tier empty rather than move past it.
|
||||
for card in rails.surprise(db, user, budget, seed, limit=SURPRISE_DRAW):
|
||||
if cards.key(card) not in skip:
|
||||
return card
|
||||
return None
|
||||
|
||||
tiers = {"resume": _resume, "comfort": _comfort, "surprise": _surprise}
|
||||
offset = start % len(HERO_TIERS)
|
||||
for tier in HERO_TIERS[offset:] + HERO_TIERS[:offset]:
|
||||
card = tiers[tier]()
|
||||
if card is not None:
|
||||
return {"tier": tier, "card": card}
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,267 @@
|
||||
"""The comfort shelf: pin a thing you can always fall back to, and have it be one click from playing.
|
||||
|
||||
Five kinds are pinnable — a video, a YouTube channel, a Plex show, a Plex movie, a playlist — because
|
||||
comfort media is almost always "put on X", a series or a channel, and only sometimes one particular
|
||||
video. `models.ComfortPin` carries that union as five nullable FKs under a `num_nonnulls(...) = 1`
|
||||
check; this module is where the union is spoken about in one vocabulary, so no call site invents a
|
||||
second one.
|
||||
|
||||
Every shelf card carries a resolved `next`: the thing that actually plays when you click the tile —
|
||||
a show's in-progress-or-next episode, a channel's newest unwatched video, a playlist's first
|
||||
unwatched item. That buys the property the whole surface exists for: one click, no intermediate page,
|
||||
no second decision. Channels and playlists are resolved for the whole shelf at once (`_Batch`), so the
|
||||
cost does not grow a query or two per pin.
|
||||
|
||||
Domain module, so nothing here raises HTTP. `resolve_ref` returns None for "no such thing, or not
|
||||
yours" and the route turns that into its 404 — core never imports routes.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.discover import cards, queries
|
||||
from app.models import Channel, ComfortPin, PlexItem, PlexShow, Playlist, User, Video
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, show_card
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
# kind -> the ComfortPin column that holds it. The client's vocabulary and the schema's, in one place.
|
||||
PIN_COLUMNS: dict[str, str] = {
|
||||
"video": "video_id",
|
||||
"channel": "channel_id",
|
||||
"plex_item": "plex_item_id",
|
||||
"plex_show": "plex_show_id",
|
||||
"playlist": "playlist_id",
|
||||
}
|
||||
PIN_KINDS = tuple(PIN_COLUMNS)
|
||||
|
||||
|
||||
def resolve_ref(db: Session, user: User, kind: str, ref: str) -> tuple[str, object] | None:
|
||||
"""Translate the client's (kind, ref) into the (column, value) the row stores, or None.
|
||||
|
||||
The client speaks the ids the rest of the API speaks: a video id, a channel id, a Plex
|
||||
`rating_key`, a playlist id. Plex's internal integer keys never leave the backend, so the
|
||||
translation belongs here rather than in five call sites.
|
||||
|
||||
The COLUMN half of every answer comes from `PIN_COLUMNS`; only the LOOKUP differs per kind
|
||||
(a direct `db.get`, a rating-key translation, an int parse plus an ownership check). Before the
|
||||
E24 review the branches spelled the column names out again and `PIN_COLUMNS` had no readers at
|
||||
all — a constant that called itself the one vocabulary while nothing spoke it."""
|
||||
col = PIN_COLUMNS.get(kind)
|
||||
if col is None:
|
||||
return None
|
||||
if kind == "video":
|
||||
return (col, ref) if db.get(Video, ref) is not None else None
|
||||
if kind == "channel":
|
||||
return (col, ref) if db.get(Channel, ref) is not None else None
|
||||
if kind == "plex_item":
|
||||
it = by_rating_key(db, PlexItem, ref)
|
||||
return (col, it.id) if it is not None else None
|
||||
if kind == "plex_show":
|
||||
sh = by_rating_key(db, PlexShow, ref)
|
||||
return (col, sh.id) if sh is not None else None
|
||||
try:
|
||||
pid = int(ref)
|
||||
# silent-ok: validating a reference the client supplied. "Not an integer" is answered as "no
|
||||
# such pin target", which is the same answer this function gives for one that does not exist.
|
||||
except ValueError:
|
||||
return None
|
||||
pl = db.get(Playlist, pid)
|
||||
# A playlist is per-user, unlike the other four (which are shared catalog rows whose PRIVACY
|
||||
# lives in the state tables). Someone else's list must read as "no such thing", not as a 403 —
|
||||
# the same reason `owned_or_404` 404s rather than 403s.
|
||||
return (col, pl.id) if pl is not None and pl.user_id == user.id else None
|
||||
|
||||
|
||||
def reorder(db: Session, user: User, ids: list[int]) -> None:
|
||||
"""Put the listed pins in the listed order, WITHIN the slots they already hold.
|
||||
|
||||
The client sends the order it can SEE, which is not always the whole shelf: with Plex off the Home
|
||||
rail hides the Plex pins, and a stale tab may not know a pin added elsewhere. Numbering only the
|
||||
listed ids 0..n-1 collided with every pin left out — two tiles on one position, and the hidden
|
||||
ones jumped about when they came back. So the pins left out keep their slots, and the listed ones
|
||||
are dealt into the slots the listed set occupies. Ids that are not this user's, or repeated, are
|
||||
ignored rather than failing a stale replay wholesale. Flushes; the route commits."""
|
||||
# Locked: the rows it leaves alone are part of the answer too. Two overlapping reorders each
|
||||
# skipped the rows that already matched in their own snapshot and interleaved into two pins on
|
||||
# one slot; the second now waits and deals from the first one's result. Locked in id order and
|
||||
# sorted HERE: with `ORDER BY position … FOR UPDATE` Postgres sorts before it locks, so the one
|
||||
# that waited got the fresh rows in its stale order and dealt the pins it left out back to their
|
||||
# old slots — and one global lock order means two reorders cannot deadlock.
|
||||
locked = db.execute(
|
||||
select(ComfortPin)
|
||||
.where(ComfortPin.user_id == user.id)
|
||||
.order_by(ComfortPin.id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalars()
|
||||
current = sorted(locked, key=lambda p: (p.position, p.id))
|
||||
owned = {p.id for p in current}
|
||||
listed: list[int] = []
|
||||
for i in ids:
|
||||
if i in owned and i not in listed:
|
||||
listed.append(i)
|
||||
chosen = set(listed)
|
||||
queue = iter(listed)
|
||||
order = [next(queue) if p.id in chosen else p.id for p in current]
|
||||
by_id = {p.id: p for p in current}
|
||||
for position, pin_id in enumerate(order):
|
||||
if by_id[pin_id].position != position:
|
||||
by_id[pin_id].position = position
|
||||
db.flush()
|
||||
|
||||
|
||||
def list_pins(db: Session, user: User) -> list[ComfortPin]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(ComfortPin)
|
||||
.where(ComfortPin.user_id == user.id)
|
||||
.order_by(ComfortPin.position.asc(), ComfortPin.id.asc())
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
def next_position(db: Session, user: User) -> int:
|
||||
"""New pins land at the end of the shelf.
|
||||
|
||||
`max(position) + 1`, not the row count. Positions are never renumbered on removal — `remove_pin`
|
||||
deletes the row, and every FK cascade silently removes a pin when the thing it points at goes
|
||||
away — so past the first hole a count under-counts and would drop the new pin into the MIDDLE of
|
||||
the shelf, or onto a position another pin already holds (nothing in the schema forbids that).
|
||||
Same idiom as `routes/playlists._next_playlist_position`."""
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.coalesce(func.max(ComfortPin.position), -1)).where(
|
||||
ComfortPin.user_id == user.id
|
||||
)
|
||||
)
|
||||
+ 1
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Batch:
|
||||
"""What the shelf resolves for every pinned channel and playlist in one go."""
|
||||
|
||||
channels: dict[str, Channel] = field(default_factory=dict)
|
||||
channel_summary: dict[str, tuple[int, dict | None]] = field(default_factory=dict)
|
||||
playlists: dict[int, Playlist] = field(default_factory=dict)
|
||||
playlist_summary: dict[int, tuple[int, dict | None]] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def load(cls, db: Session, user: User, pins: list[ComfortPin]) -> "_Batch":
|
||||
ch_ids = [p.channel_id for p in pins if p.channel_id is not None]
|
||||
pl_ids = [p.playlist_id for p in pins if p.playlist_id is not None]
|
||||
b = cls()
|
||||
if ch_ids:
|
||||
b.channels = {
|
||||
c.id: c for c in db.scalars(select(Channel).where(Channel.id.in_(ch_ids)))
|
||||
}
|
||||
b.channel_summary = queries.channels_summary(db, user, ch_ids)
|
||||
if pl_ids:
|
||||
b.playlists = {
|
||||
p.id: p for p in db.scalars(select(Playlist).where(Playlist.id.in_(pl_ids)))
|
||||
}
|
||||
b.playlist_summary = queries.playlists_summary(db, user, pl_ids)
|
||||
return b
|
||||
|
||||
|
||||
def _shelf_card(db: Session, user: User, pin: ComfortPin, batch: _Batch) -> dict | None:
|
||||
"""One pin as a unified card with its resolved `next`. None when the pinned row has gone (the FK
|
||||
cascade removes the pin itself, so this is only the window between a delete and a refetch)."""
|
||||
card: dict | None = None
|
||||
nxt: dict | None = None
|
||||
|
||||
if pin.video_id is not None:
|
||||
row = queries.yt_video_row(db, user, pin.video_id)
|
||||
if row is None:
|
||||
return None
|
||||
card = cards.from_video(row)
|
||||
nxt = dict(card) # a video plays itself
|
||||
|
||||
elif pin.channel_id is not None:
|
||||
ch = batch.channels.get(pin.channel_id)
|
||||
if ch is None:
|
||||
return None
|
||||
count, row = batch.channel_summary.get(ch.id, (0, None))
|
||||
card = cards.from_channel(ch, count)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
elif pin.plex_item_id is not None:
|
||||
it = db.get(PlexItem, pin.plex_item_id)
|
||||
if it is None:
|
||||
return None
|
||||
st = rollup.item_state_map(db, user.id, [it.id]).get(it.id)
|
||||
card = cards.from_plex(leaf_card(it, st))
|
||||
nxt = dict(card) # a movie or a single episode plays itself
|
||||
|
||||
elif pin.plex_show_id is not None:
|
||||
sh = db.get(PlexShow, pin.plex_show_id)
|
||||
if sh is None:
|
||||
return None
|
||||
status = rollup.state_map(db, user.id, [sh.id]).get(sh.id, "new")
|
||||
card = cards.from_plex(show_card(sh, status))
|
||||
ep = queries.next_episode(db, user.id, sh.id)
|
||||
nxt = cards.from_plex(ep) if ep else None
|
||||
|
||||
elif pin.playlist_id is not None:
|
||||
pl = batch.playlists.get(pin.playlist_id)
|
||||
if pl is None:
|
||||
return None
|
||||
count, row = batch.playlist_summary.get(pl.id, (0, None))
|
||||
card = cards.from_playlist(pl, count, row.get("thumbnail_url") if row else None)
|
||||
nxt = cards.from_video(row) if row else None
|
||||
|
||||
if card is None:
|
||||
return None
|
||||
# A COPY, never the card itself: a self-playing pin (a video, a movie) would otherwise make
|
||||
# `card["next"] is card`, which serializes as a circular reference and 500s the whole shelf. The
|
||||
# copy is taken before `pin_id`/`next` are attached, so `next` stays a plain playable card.
|
||||
card["pin_id"] = pin.id
|
||||
card["next"] = nxt
|
||||
return card
|
||||
|
||||
|
||||
_PLEX_COLUMNS = ("plex_item_id", "plex_show_id")
|
||||
|
||||
|
||||
def shelf(db: Session, user: User, *, include_plex: bool = True) -> list[dict]:
|
||||
"""The whole shelf, in the user's own order.
|
||||
|
||||
`include_plex=False` is the Home surface with the deployment's Plex toggle off (the owner's call,
|
||||
2026-09-19): the rails beside it go YouTube-only, and a Plex tile there would offer a Play that
|
||||
lands on a gated module. The pins themselves are kept — the pin menus still read the full shelf
|
||||
(`GET /pins`), and switching Plex back on brings the tiles back in their old slots
|
||||
(`reorder` keeps a hidden pin's place)."""
|
||||
out = []
|
||||
pins = list_pins(db, user)
|
||||
if not include_plex:
|
||||
pins = [p for p in pins if all(getattr(p, c) is None for c in _PLEX_COLUMNS)]
|
||||
batch = _Batch.load(db, user, pins)
|
||||
for pin in pins:
|
||||
card = _shelf_card(db, user, pin, batch)
|
||||
if card is not None:
|
||||
out.append(card)
|
||||
return out
|
||||
|
||||
|
||||
def playables(shelf_cards: list[dict]) -> list[dict]:
|
||||
"""The `next` of every shelf entry that has one — what the hero may draw from. A pinned channel
|
||||
with nothing unwatched left, or an empty playlist, simply is not an answer right now.
|
||||
|
||||
Each carries `pinned_from`: the pin it was resolved from. A show's next EPISODE is not itself
|
||||
pinned, so without it the hero's menu looked the episode up, missed, and offered "Pin" under a
|
||||
kicker that says "From your shelf" — and taking it inserted a second pin. With it, the hero's
|
||||
pin row acts on the shelf entry the card came from (the owner's call, 2026-09-19: "Unpin")."""
|
||||
return [
|
||||
{
|
||||
**c["next"],
|
||||
"pinned_from": {k: c[k] for k in ("source", "kind", "id", "title")},
|
||||
}
|
||||
for c in shelf_cards
|
||||
if c.get("next")
|
||||
]
|
||||
@@ -0,0 +1,361 @@
|
||||
"""The row fetchers behind the Home surface — one place where "what is startable right now" is
|
||||
expressed as SQL, for both sources.
|
||||
|
||||
Two rules shape this module:
|
||||
|
||||
**Reuse the query layer, not the route.** The YouTube half calls `feedquery.filtered_query` /
|
||||
`sort_keys`, so the Home surface inherits the feed's own definitions of subscribed, blocked,
|
||||
explore-visible and in-progress for free, and cannot drift from them. The Plex half calls
|
||||
`plex.rollup` and the `plex.cards` serializers rather than `routes/plex/discover.py`: that route
|
||||
narrows EVERY rail by the Plex sidebar's `LibraryFilters`, which is right for a surface that HAS a
|
||||
sidebar and wrong for a landing page that does not.
|
||||
|
||||
**One seed drives both halves.** The feed orders a shuffle by `md5(id || seed)` so a keyset cursor
|
||||
stays valid across pages; the Plex surprise rail uses `ORDER BY random()`, which is fine for a single
|
||||
un-paged draw but cannot be reproduced. A cross-source reroll needs both halves to move together —
|
||||
otherwise "Mást kérek" re-draws the Plex picks while the YouTube ones stand still — so the Plex side
|
||||
here takes the feed's seeded-md5 idiom instead of `random()`.
|
||||
"""
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.discover.budget import band
|
||||
from app.feedquery import (
|
||||
feed_columns,
|
||||
filtered_query,
|
||||
not_blocked_cond,
|
||||
order_by,
|
||||
serialize,
|
||||
sort_keys,
|
||||
)
|
||||
from app.models import (
|
||||
Channel,
|
||||
PlaylistItem,
|
||||
PlexItem,
|
||||
PlexShow,
|
||||
PlexState,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import leaf_card, movie_card, show_card
|
||||
from app.plex.query import apply_show_state_filter, content_libs
|
||||
|
||||
# How many rows a single rail may carry — Home's rails and the Plex module's Discover rails alike
|
||||
# (`routes/plex/discover.py` imports it): the two surfaces should feel like the same app, and a rail
|
||||
# is a rail. One definition, because a comment was all that held two copies in step.
|
||||
RAIL_LIMIT = 20
|
||||
|
||||
# The feed parameters a Home rail runs with. Spelled out rather than borrowed from the route's
|
||||
# `_feed_params` defaults, because those are an HTTP contract that may change for the feed's own
|
||||
# reasons; the landing page's scope should not move with them silently.
|
||||
_FEED_BASE: dict = {
|
||||
"tags": [],
|
||||
"tag_modes": {},
|
||||
"untagged": False,
|
||||
"channel_ids": [],
|
||||
"q": None,
|
||||
"min_duration": None,
|
||||
"max_duration": None,
|
||||
"max_age_days": None,
|
||||
"published_after": None,
|
||||
"published_before": None,
|
||||
"show_normal": True,
|
||||
"include_shorts": False,
|
||||
"include_live": False,
|
||||
"show": "unwatched",
|
||||
"scope": "my",
|
||||
"library_source": "organic",
|
||||
}
|
||||
|
||||
|
||||
def _seeded(col, seed: int):
|
||||
"""The feed's deterministic shuffle expression, reused for Plex so one seed moves both halves."""
|
||||
return func.md5(func.concat(col, str(seed)))
|
||||
|
||||
|
||||
# ── YouTube ───────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def yt_in_progress(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[tuple[object, dict]]:
|
||||
"""Started-but-not-finished videos, most recently progressed first, as (progressed_at, row).
|
||||
|
||||
The predicate is `feedquery`'s `show="in_progress"` verbatim (a stored resume position, status not
|
||||
watched/hidden) — but the query is built here rather than through `filtered_query` because this
|
||||
rail wants an ordering by `progress_updated_at`, and that column lives on the state alias the feed
|
||||
query keeps private. Driving from `video_states` is also the faster plan: it is a per-user table of
|
||||
a few hundred rows, against a catalog of many thousands.
|
||||
|
||||
Building it here means the feed's OTHER predicates have to be chosen deliberately rather than
|
||||
inherited. `not_blocked_cond` is shared, because "never show me this channel" outranks "you
|
||||
started it" — blocking leaves the state row untouched, so without this the video the user just
|
||||
banned heads the resume rail. The feed's SCOPE clauses (subscribed / via_explore accessibility)
|
||||
are deliberately NOT applied, for the reason `yt_video_row` gives: something you have already
|
||||
started must stay resumable even after you unsubscribe or an exploration expires."""
|
||||
st = aliased(VideoState)
|
||||
q = (
|
||||
select(*feed_columns(user, st), st.progress_updated_at.label("progressed_at"))
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.join(st, and_(st.video_id == Video.id, st.user_id == user.id))
|
||||
.where(
|
||||
st.position_seconds > 0,
|
||||
st.status.notin_(("watched", "hidden")),
|
||||
Video.unavailable_since.is_(None),
|
||||
not_blocked_cond(user),
|
||||
)
|
||||
.order_by(st.progress_updated_at.desc().nullslast())
|
||||
.limit(limit)
|
||||
)
|
||||
return [(r.progressed_at, serialize(r)) for r in db.execute(q).all()]
|
||||
|
||||
|
||||
def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) -> list[dict]:
|
||||
"""Unwatched videos from the user's own feed scope, in seeded-shuffle order, inside the budget.
|
||||
|
||||
Shorts are opened up only for the shortest band: "I have ten minutes" is the one answer a Short is
|
||||
actually a good response to, and the feed's own default keeps them out otherwise."""
|
||||
lo, hi = band(budget)
|
||||
params = dict(_FEED_BASE, min_duration=lo, max_duration=hi, include_shorts=budget == "short")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys("shuffle", seed, rank_expr)
|
||||
# Over-fetch, then drop anything already started. The feed's `unwatched` scope keeps watched and
|
||||
# hidden out but deliberately keeps in-progress rows in (they belong in the feed). Here they must
|
||||
# go: the resume tier has already had its chance at them, and it may have passed precisely because
|
||||
# the REMAINING time did not fit the budget — offering the same item back as a surprise, whose
|
||||
# runtime does fit, would label a half-finished thing "Meglepetés" and contradict the question.
|
||||
rows = db.execute(query.order_by(*order_by(keys)).limit(limit * 4)).all()
|
||||
out = [serialize(r) for r in rows]
|
||||
return [r for r in out if not r["position_seconds"]][:limit]
|
||||
|
||||
|
||||
def yt_video_row(db: Session, user: User, video_id: str) -> dict | None:
|
||||
"""One video as a feed row, with this user's watch state.
|
||||
|
||||
Built directly rather than through `filtered_query` on purpose: a PINNED video must keep working
|
||||
even when it falls outside the feed's scope — the user unsubscribed from the channel, or blocked
|
||||
it, or the video only ever arrived through a search. The shelf is the surface you fall back to;
|
||||
it silently losing an entry because a filter moved is exactly the failure it must not have."""
|
||||
st = aliased(VideoState)
|
||||
row = db.execute(
|
||||
select(*feed_columns(user, st))
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(st, and_(st.video_id == Video.id, st.user_id == user.id))
|
||||
.where(Video.id == video_id)
|
||||
).first()
|
||||
return serialize(row) if row is not None else None
|
||||
|
||||
|
||||
def channels_summary(
|
||||
db: Session, user: User, channel_ids: list[str]
|
||||
) -> dict[str, tuple[int, dict | None]]:
|
||||
"""For each pinned channel: (unwatched count, its newest unwatched video) — the tile's only honest
|
||||
promise, since a channel has no runtime of its own, and what "put on that channel" resolves to.
|
||||
|
||||
ONE query for the whole shelf, not two per channel: the count and the first row per channel are
|
||||
window functions over the same filtered set. `library_source="all"`: a pinned channel means the
|
||||
whole channel, including videos that only ever arrived through a search — the feed's "organic"
|
||||
default is a feed-scoping choice, not a statement about what belongs to a channel."""
|
||||
if not channel_ids:
|
||||
return {}
|
||||
params = dict(_FEED_BASE, channel_ids=list(channel_ids), scope="all", library_source="all")
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
newest = order_by(sort_keys("newest", 0, rank_expr))
|
||||
per_channel = {"partition_by": Video.channel_id}
|
||||
sub = query.add_columns(
|
||||
func.row_number().over(**per_channel, order_by=newest).label("_rn"),
|
||||
func.count().over(**per_channel).label("_total"),
|
||||
).subquery()
|
||||
rows = db.execute(select(sub).where(sub.c._rn == 1)).all()
|
||||
return {r.channel_id: (int(r._total), serialize(r)) for r in rows}
|
||||
|
||||
|
||||
def playlists_summary(
|
||||
db: Session, user: User, playlist_ids: list[int]
|
||||
) -> dict[int, tuple[int, dict | None]]:
|
||||
"""For each pinned playlist: (item count, the row to play next) — its first not-yet-watched item,
|
||||
or its first item once the whole list is watched. The same row doubles as the tile's artwork, so
|
||||
a shelf of playlists is scannable instead of a row of blank boxes. One query for the whole shelf."""
|
||||
if not playlist_ids:
|
||||
return {}
|
||||
st = aliased(VideoState)
|
||||
rows = db.execute(
|
||||
select(*feed_columns(user, st), PlaylistItem.playlist_id.label("_playlist"))
|
||||
.join(PlaylistItem, PlaylistItem.video_id == Video.id)
|
||||
.join(Channel, Channel.id == Video.channel_id)
|
||||
.outerjoin(st, and_(st.video_id == Video.id, st.user_id == user.id))
|
||||
.where(PlaylistItem.playlist_id.in_(playlist_ids))
|
||||
.order_by(PlaylistItem.playlist_id, PlaylistItem.position.asc())
|
||||
).all()
|
||||
grouped: dict[int, list] = {}
|
||||
for r in rows:
|
||||
grouped.setdefault(r._playlist, []).append(r)
|
||||
out: dict[int, tuple[int, dict | None]] = {}
|
||||
for pid, items in grouped.items():
|
||||
nxt = next((r for r in items if (r.status or "new") not in ("watched", "hidden")), items[0])
|
||||
out[pid] = (len(items), serialize(nxt))
|
||||
return out
|
||||
|
||||
|
||||
# ── Plex ──────────────────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _unseen_movie_conds(uid: int):
|
||||
"""A movie the user has neither finished nor hidden. A LEFT-JOIN-IS-NULL would also drop rows that
|
||||
merely carry a resume position, which are exactly the ones Continue watching wants back."""
|
||||
seen = (
|
||||
select(1)
|
||||
.select_from(PlexState)
|
||||
.where(
|
||||
PlexState.item_id == PlexItem.id,
|
||||
PlexState.user_id == uid,
|
||||
or_(
|
||||
PlexState.status.in_(("watched", "hidden")),
|
||||
# Started counts as "not a surprise" too — see the note in `yt_surprise`.
|
||||
PlexState.position_seconds > 0,
|
||||
),
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
return ~seen
|
||||
|
||||
|
||||
def plex_in_progress(db: Session, uid: int, limit: int = RAIL_LIMIT) -> list[tuple[object, dict]]:
|
||||
"""In-progress movies + partially-watched shows as (progressed_at, card).
|
||||
|
||||
Mirrors `routes/plex/discover.py::_continue_watching` — same `PlexState` predicate for movies (a
|
||||
NULL status still counts as started, so `notin_` alone would drop it), and the same shared
|
||||
`rollup.agg_subquery` + `apply_show_state_filter` for shows, so the two rails cannot disagree on
|
||||
what "in progress" means. What it does NOT inherit is that route's `LibraryFilters` narrowing:
|
||||
this surface has no Plex sidebar to read a filter from."""
|
||||
movie_libs, show_libs = content_libs(db)
|
||||
out: list[tuple[object, dict]] = []
|
||||
|
||||
if movie_libs:
|
||||
mq = (
|
||||
select(PlexItem, PlexState)
|
||||
.join(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
|
||||
.where(
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.library_id.in_(movie_libs),
|
||||
PlexState.position_seconds > 0,
|
||||
or_(
|
||||
PlexState.status.is_(None),
|
||||
PlexState.status.notin_(("watched", "hidden")),
|
||||
),
|
||||
)
|
||||
.order_by(PlexState.progress_updated_at.desc().nullslast())
|
||||
.limit(limit)
|
||||
)
|
||||
for it, st in db.execute(mq).all():
|
||||
out.append((st.progress_updated_at, movie_card(it, st)))
|
||||
|
||||
if show_libs:
|
||||
prog = rollup.agg_subquery(db, uid, include_resumed=True).subquery()
|
||||
sq = apply_show_state_filter(
|
||||
select(PlexShow, prog.c.resumed_at).join(prog, prog.c.show_id == PlexShow.id),
|
||||
prog.c.total,
|
||||
prog.c.watched,
|
||||
prog.c.inprog,
|
||||
"in_progress",
|
||||
).where(PlexShow.library_id.in_(show_libs))
|
||||
for sh, resumed_at in db.execute(
|
||||
sq.order_by(prog.c.resumed_at.desc().nullslast()).limit(limit)
|
||||
).all():
|
||||
out.append((resumed_at, show_card(sh, "in_progress")))
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def plex_surprise(db: Session, uid: int, budget: str, seed: int, limit: int) -> list[dict]:
|
||||
"""Unseen playable movies inside the budget, in seeded-shuffle order.
|
||||
|
||||
Movies only. A show is not a thing you can start playing, and resolving every candidate show to
|
||||
its next episode just to draw three cards would be a query per row for a rail whose whole value is
|
||||
that it answers instantly — shows reach this surface through Continue watching and the comfort
|
||||
shelf, which is where "put the next episode on" actually belongs.
|
||||
|
||||
No `playable` filter, matching the Plex surprise rail: `PlexItem.playable` is a playback CLASS
|
||||
("direct" / "remux" / "transcode"), not a yes/no, and NULL there means "not probed yet" rather than
|
||||
"will not play". Excluding un-probed items would quietly hide a fresh library from the one surface
|
||||
that is supposed to have an answer. The class travels on the card, so the client still shows it."""
|
||||
movie_libs, _ = content_libs(db)
|
||||
if not movie_libs:
|
||||
return []
|
||||
lo, hi = band(budget)
|
||||
q = (
|
||||
select(PlexItem)
|
||||
.where(
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.library_id.in_(movie_libs),
|
||||
_unseen_movie_conds(uid),
|
||||
)
|
||||
.order_by(_seeded(PlexItem.rating_key, seed))
|
||||
.limit(limit)
|
||||
)
|
||||
if lo is not None:
|
||||
q = q.where(PlexItem.duration_s >= lo)
|
||||
if hi is not None:
|
||||
q = q.where(PlexItem.duration_s <= hi)
|
||||
items = db.execute(q).scalars().all()
|
||||
states = rollup.item_state_map(db, uid, [it.id for it in items])
|
||||
return [movie_card(it, states.get(it.id)) for it in items]
|
||||
|
||||
|
||||
# Watch states that take an episode out of the running for "what would play next".
|
||||
_DONE = ("watched", "hidden")
|
||||
|
||||
|
||||
def next_episode(db: Session, uid: int, show_id: int) -> dict | None:
|
||||
"""What "put that show on" means: the episode in progress, else the first unwatched one, else the
|
||||
first — over the same `(season, episode)` broadcast ordering `detail.py::_episode_nav` uses.
|
||||
|
||||
Three ordered `LIMIT 1` probes rather than one scan picked apart in Python. The Python version
|
||||
loaded EVERY episode of the show to choose one of them — 236 rows for Friends, 227 for JAG — which
|
||||
the comfort shelf could just about afford (a handful of hand-picked pins) and the resume rail
|
||||
could not, since it resolves a show per rail row on the landing page.
|
||||
|
||||
Returned as a `leaf_card` so it carries the show title the episode row itself does not."""
|
||||
asc = (
|
||||
PlexItem.season_number.asc().nullsfirst(),
|
||||
PlexItem.episode_number.asc().nullsfirst(),
|
||||
)
|
||||
base = (
|
||||
select(PlexItem, PlexState)
|
||||
.outerjoin(PlexState, and_(PlexState.item_id == PlexItem.id, PlexState.user_id == uid))
|
||||
.where(PlexItem.show_id == show_id, PlexItem.kind == "episode")
|
||||
.limit(1)
|
||||
)
|
||||
# In progress — the LAST one in broadcast order, which is `plex.rollup.rollup`'s `inprog[-1]`,
|
||||
# i.e. what the show page's own Resume button plays. Two episodes of one show are in progress at
|
||||
# once more often than it sounds (`watch_sync._reconcile_inprogress_movies` documents that a
|
||||
# scrub to the end leaves a stale `status='new', position>0` row until the daily reconcile), and
|
||||
# taking opposite ends of that set is how one show came to have two different "continue" answers:
|
||||
# the Home tile ranked it by its LATEST touch and then offered the EARLIEST started episode.
|
||||
# `position_seconds > 0` already requires the state row to exist, so no null guard.
|
||||
pick = db.execute(
|
||||
base.where(PlexState.position_seconds > 0, PlexState.status.notin_(_DONE)).order_by(
|
||||
PlexItem.season_number.desc().nullslast(),
|
||||
PlexItem.episode_number.desc().nullslast(),
|
||||
)
|
||||
).first()
|
||||
# Unwatched — INCLUDING an episode with no state row at all, which is the common case and the one
|
||||
# a bare `status NOT IN (...)` silently drops (NULL is not false, it is unknown, so the row fails
|
||||
# the filter).
|
||||
pick = (
|
||||
pick
|
||||
or db.execute(
|
||||
base.where(or_(PlexState.id.is_(None), PlexState.status.notin_(_DONE))).order_by(*asc)
|
||||
).first()
|
||||
)
|
||||
# Fully watched show: start it again from the top.
|
||||
pick = pick or db.execute(base.order_by(*asc)).first()
|
||||
if pick is None:
|
||||
return None
|
||||
|
||||
ep, st = pick
|
||||
show = db.get(PlexShow, show_id)
|
||||
return leaf_card(
|
||||
ep, st, show_title=show.title if show else None, show_rk=show.rating_key if show else None
|
||||
)
|
||||
@@ -0,0 +1,97 @@
|
||||
"""The Home surface's rails: cross-source lists, assembled from `queries.py` and emitted as unified
|
||||
cards.
|
||||
|
||||
The whole point of this module is that a rail is ONE list, not a YouTube list beside a Plex list. Two
|
||||
side-by-side rails would just move the choice the user could not make ("which half do I look at?") one
|
||||
level up, which is the opposite of what this epic is for.
|
||||
"""
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.discover import cards, queries
|
||||
from app.discover.queries import RAIL_LIMIT
|
||||
from app.models import PlexShow, User
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.plex.rollup import NO_TS
|
||||
|
||||
|
||||
def plex_available(db: Session) -> bool:
|
||||
"""Plex is a deployment-level toggle, not a per-user one (see `routes/me.py`). With it off, the
|
||||
Home rails are YouTube-only and say nothing about Plex."""
|
||||
return sysconfig.get_bool(db, "plex_enabled")
|
||||
|
||||
|
||||
def playable(db: Session, uid: int, card: dict) -> dict | None:
|
||||
"""The thing that would actually START from this card — itself for anything with a runtime, and
|
||||
for a Plex SHOW the episode that is next (in progress, else the first unwatched, else the first).
|
||||
|
||||
ONE function, because three surfaces need the same answer and a second implementation is how the
|
||||
shelf and the rail came to disagree in the first place: the hero calls it to decide what it is
|
||||
offering, `continue_watching` calls it to fill each show tile's `next`, and the comfort shelf
|
||||
resolves the same way in `pins`. None when the show has no episodes mirrored yet.
|
||||
|
||||
A card that already carries a resolved `next` (a rail row) is taken at its word rather than
|
||||
re-queried — the hero reads the rail it was handed."""
|
||||
if not (card.get("source") == "plex" and card.get("kind") == "show"):
|
||||
return card
|
||||
if "next" in card:
|
||||
return card["next"]
|
||||
sh = by_rating_key(db, PlexShow, card["id"])
|
||||
if sh is None:
|
||||
return None
|
||||
ep = queries.next_episode(db, uid, sh.id)
|
||||
return cards.from_plex(ep) if ep else None
|
||||
|
||||
|
||||
def continue_watching(db: Session, user: User, limit: int = RAIL_LIMIT) -> list[dict]:
|
||||
"""One resume rail across both sources, most recently progressed first.
|
||||
|
||||
A Plex SHOW enters as a show CARD — that is what the Plex rail shows, and it is the artwork the
|
||||
user recognises — but it carries `next`: the episode that would actually start. Without it a
|
||||
click on a half-watched series landed on the show page with six seasons to choose between, which
|
||||
is the decision this whole surface exists to remove (the comfort shelf and the hero already
|
||||
resolved; only this rail did not). The client plays `next ?? card`, so the tile stays a poster and
|
||||
the click stays one click."""
|
||||
rows: list[tuple[object, dict]] = [
|
||||
(ts, cards.from_video(r)) for ts, r in queries.yt_in_progress(db, user, limit)
|
||||
]
|
||||
if plex_available(db):
|
||||
rows += [(ts, cards.from_plex(c)) for ts, c in queries.plex_in_progress(db, user.id, limit)]
|
||||
rows.sort(key=lambda r: r[0] or NO_TS, reverse=True)
|
||||
out = [card for _, card in rows[:limit]]
|
||||
# Resolve AFTER the slice: only the rows that survive it are worth a query.
|
||||
for card in out:
|
||||
if card["kind"] == "show":
|
||||
card["next"] = playable(db, user.id, card)
|
||||
return out
|
||||
|
||||
|
||||
def surprise(db: Session, user: User, budget: str, seed: int, limit: int = 3) -> list[dict]:
|
||||
"""The hero's draw pool: unseen things that fit the budget, from both sources under ONE seed.
|
||||
|
||||
Interleaved rather than concatenated — the same alternation `routes/plex/discover.py` uses for its
|
||||
movie/show split — so a short pool never comes back all-YouTube or all-Plex just because one side
|
||||
happened to be queried first.
|
||||
|
||||
WHICH source leads is decided by the seed, not by a hardcoded order. The hero draws a pool of ONE,
|
||||
so a fixed order would mean the Plex library could never answer "egy este" while a single YouTube
|
||||
video also fit — the reroll would look broken and the cross-source promise would quietly be a
|
||||
YouTube promise. Parity on the same seed that orders the rows keeps it deterministic: the same
|
||||
seed always gives the same answer, and pressing "Mást kérek" moves both halves together."""
|
||||
halves = [
|
||||
(cards.from_video, queries.yt_surprise(db, user, budget, seed, limit)),
|
||||
(
|
||||
cards.from_plex,
|
||||
queries.plex_surprise(db, user.id, budget, seed, limit) if plex_available(db) else [],
|
||||
),
|
||||
]
|
||||
if seed % 2:
|
||||
halves.reverse()
|
||||
|
||||
out: list[dict] = []
|
||||
for i in range(max(len(rows) for _, rows in halves)):
|
||||
for to_card, rows in halves:
|
||||
if i < len(rows):
|
||||
out.append(to_card(rows[i]))
|
||||
return out[:limit]
|
||||
@@ -11,9 +11,22 @@ Resolution works both before extraction (from source_kind / URL host) and after
|
||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||
"""
|
||||
|
||||
import re
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from yt_dlp.networking.impersonate import ImpersonateTarget
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads.stated import StatedFailure
|
||||
|
||||
# Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry
|
||||
# below and the failure catalog (errors.py) decide on it, so the wording exists once.
|
||||
FACEBOOK_PARSE_WALL = "cannot parse data"
|
||||
# A source's own verdicts on a link, raised from `resolve_url`. errors.py classifies them; it imports
|
||||
# them from here because this module must not import errors (errors → credentials → adapters).
|
||||
POST_WITHOUT_VIDEO_MESSAGE = "No single video was found in this post."
|
||||
POST_NOT_SHOWN_MESSAGE = "Facebook did not show this post to the signed-in session."
|
||||
|
||||
|
||||
class SourceAdapter:
|
||||
@@ -22,9 +35,21 @@ class SourceAdapter:
|
||||
key = "generic"
|
||||
|
||||
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||||
def client_sets(self) -> list:
|
||||
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
|
||||
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
"""The link to hand yt-dlp for `url`, resolved BEFORE the first attempt. `ydl_opts` are the
|
||||
download's own options (cookie, timeouts, address family) for any page this needs to read.
|
||||
|
||||
Raises a RuntimeError carrying one of this module's verdicts when the page read shows the
|
||||
download cannot go ahead: the content is there but holds no single video
|
||||
(`POST_WITHOUT_VIDEO_MESSAGE`), or the page is not the content at all — the login page a
|
||||
refused cookie gets (`POST_NOT_SHOWN_MESSAGE`), never the no-video verdict. A failed page read
|
||||
is raised as itself, so it is classified like any other download error. Generic: `url`."""
|
||||
return url
|
||||
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
"""Attempts to make, in order — each entry is handed back to `ydl_opts_extra`, and the next one
|
||||
is tried when `retriable` says so. A generic source makes a single default attempt (`[None]`).
|
||||
`authed` is whether the acting user's cookie file goes with the download."""
|
||||
return [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
@@ -37,6 +62,10 @@ class SourceAdapter:
|
||||
return False
|
||||
|
||||
# --- metadata -----------------------------------------------------------------------------
|
||||
def title(self, info: dict) -> str | None:
|
||||
"""The download's raw title, before `titles.normalize_title` cleans it. Generic: yt-dlp's."""
|
||||
return info.get("title")
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||||
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||||
@@ -65,12 +94,46 @@ class YouTubeAdapter(SourceAdapter):
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
# Measured 2026-09-16: a gated video whose EXTRACTION succeeds and whose media fetch is then
|
||||
# refused by the CDN ("unable to download video data: HTTP Error 403"). The wall moves
|
||||
# between the player response and the stream, so this is the same per-client flakiness one
|
||||
# step later — and it used to end the job on attempt 1, because the text classifies as
|
||||
# `network` and nothing here matched it. The format URLs belong to the client that minted
|
||||
# them, so the next client set is a real second chance.
|
||||
"unable to download video data",
|
||||
)
|
||||
# Measured in the worker against a REAL account cookie (2026-09-16, yt-dlp 2026.07.04): with
|
||||
# `player_client=android_vr` yt-dlp logs `Skipping client "android_vr" since it does not support
|
||||
# cookies`, drops it, and the request that is left with no client fails with "Requested format is
|
||||
# not available" — which is already a retriable marker above, so the cost is a wasted attempt
|
||||
# rather than a hard failure. Our primary set is android_vr ALONE, so without this filter every
|
||||
# signed-in download would burn its first attempt that way before reaching the POT-backed web
|
||||
# fallback the cookie exists for. The list is yt-dlp's own INNERTUBE_CLIENTS[...]
|
||||
# ["SUPPORTS_COOKIES"], pinned by test_adapters against the installed yt-dlp, so a version bump
|
||||
# that moves the table turns the gate red instead of silently un-authing every YouTube download.
|
||||
_COOKIE_CAPABLE = frozenset(
|
||||
{
|
||||
"mweb",
|
||||
"tv",
|
||||
"tv_downgraded",
|
||||
"web",
|
||||
"web_creator",
|
||||
"web_embedded",
|
||||
"web_music",
|
||||
"web_safari",
|
||||
}
|
||||
)
|
||||
# Where a signed-in download lands when the admin's knobs hold no cookie-capable client at all.
|
||||
# Both clients in ONE set: yt-dlp queries each and merges their formats, which is the cover for
|
||||
# the per-client flakiness that the two-tier ladder gives a signed-out download.
|
||||
_AUTHED_DEFAULT = ("web_safari", "web")
|
||||
|
||||
def client_sets(self) -> list:
|
||||
return [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
sets = [c for c in (settings.player_client_list, settings.player_client_fallback_list) if c]
|
||||
if authed:
|
||||
kept = [[c for c in s if c in self._COOKIE_CAPABLE] for s in sets]
|
||||
return [s for s in kept if s] or [list(self._AUTHED_DEFAULT)]
|
||||
return sets or [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||||
@@ -93,6 +156,63 @@ class YouTubeAdapter(SourceAdapter):
|
||||
|
||||
class FacebookAdapter(SourceAdapter):
|
||||
key = "facebook"
|
||||
# Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||||
# with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a
|
||||
# SECOND attempt behind that one error, not the default: the links that already download with the
|
||||
# cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without
|
||||
# a cookie the same error is the login wall, which no disguise gets past.
|
||||
_IMPERSONATE = "impersonate"
|
||||
# A post page is ~4–5 MB of inline JSON; anything far past that is not a post page.
|
||||
_POST_PAGE_MAX_BYTES = 16 * 1024 * 1024
|
||||
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
# Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
|
||||
# downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
|
||||
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
|
||||
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
|
||||
post_id = facebook_group_post_id(url)
|
||||
if post_id is None or not ydl_opts.get("cookiefile"):
|
||||
return url
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
|
||||
page = raw.decode("utf-8", "replace")
|
||||
# Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
|
||||
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
|
||||
if _quoted(post_id) not in page:
|
||||
raise StatedFailure(POST_NOT_SHOWN_MESSAGE)
|
||||
video_id = facebook_post_video_id(page, post_id)
|
||||
if video_id is None:
|
||||
# Never fall back to the post link: that is exactly where the wrong video comes from.
|
||||
raise StatedFailure(POST_WITHOUT_VIDEO_MESSAGE)
|
||||
return f"https://www.facebook.com/watch/?v={video_id}"
|
||||
|
||||
def client_sets(self, authed: bool = False) -> list:
|
||||
return [None, self._IMPERSONATE] if authed else [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
if clients == self._IMPERSONATE:
|
||||
return {"impersonate": ImpersonateTarget.from_str("chrome")}
|
||||
return {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
return FACEBOOK_PARSE_WALL in error_text.lower()
|
||||
|
||||
# Measured (2026-09-14): a SIGNED-IN read gives the placeholder "Video" (plain and as Chrome), and an
|
||||
# 18+ reel only readable as Chrome gives "Facebook"; the post's text is still in `description`. A
|
||||
# signed-out read titles the page with that same text behind the engagement counts, cut short.
|
||||
_PLACEHOLDER_TITLES = frozenset({"video", "facebook"})
|
||||
|
||||
def title(self, info: dict) -> str | None:
|
||||
raw = info.get("title")
|
||||
if (raw or "").strip().lower() not in self._PLACEHOLDER_TITLES:
|
||||
return raw
|
||||
line = next(
|
||||
(ln.strip() for ln in (info.get("description") or "").splitlines() if ln.strip()), ""
|
||||
)
|
||||
# The whole first line, never cut here with an ellipsis: normalize_title's NFKC turns "…" into
|
||||
# "..." and then collapses it to ".", which read as a finished sentence ("… live at."). The
|
||||
# stored title is unbounded; display_name and the file name clip it themselves.
|
||||
return line or raw # no description either (the 18+ reel): nothing better on the page
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||||
@@ -107,6 +227,39 @@ class FacebookAdapter(SourceAdapter):
|
||||
return True
|
||||
|
||||
|
||||
_FB_GROUP_POST_PATH = re.compile(r"^/groups/[^/]+/(?:permalink|posts)/(\d+)/?$")
|
||||
_SCRIPT_BLOCK = re.compile(r"<script[^>]*>(.*?)</script>", re.S)
|
||||
_VIDEO_NODE = re.compile(r'"__typename":"Video","id":"(\d+)"')
|
||||
|
||||
|
||||
def _quoted(post_id: str) -> str:
|
||||
# How the page JSON carries the id: quoted, so a longer id that merely starts with it never counts.
|
||||
return f'"{post_id}"'
|
||||
|
||||
|
||||
def facebook_group_post_id(url: str) -> str | None:
|
||||
"""The post id of a Facebook group post link (`/groups/<group>/permalink|posts/<id>/`), else None."""
|
||||
parsed = urlparse(url)
|
||||
if not _host_is(parsed.netloc.lower(), "facebook.com"):
|
||||
return None
|
||||
match = _FB_GROUP_POST_PATH.match(parsed.path)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def facebook_post_video_id(page: str, post_id: str) -> str | None:
|
||||
"""The id of the one video attached to `post_id`, read from its page: the Video nodes of the script
|
||||
blocks that carry the post id (the page's other ~10 videos — feed, reels — sit in other blocks).
|
||||
None for no video AND for more than one: guessing between two would be the wrong-video bug again."""
|
||||
quoted = _quoted(post_id)
|
||||
ids = {
|
||||
vid
|
||||
for block in _SCRIPT_BLOCK.findall(page)
|
||||
if quoted in block
|
||||
for vid in _VIDEO_NODE.findall(block)
|
||||
}
|
||||
return ids.pop() if len(ids) == 1 else None
|
||||
|
||||
|
||||
class RedditAdapter(SourceAdapter):
|
||||
key = "reddit"
|
||||
|
||||
|
||||
@@ -11,11 +11,12 @@ Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
@@ -23,12 +24,17 @@ from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
||||
needs the POT/player-client strategy even just to extract)."""
|
||||
with formats.hiding(url):
|
||||
return _extract_meta_scoped(url, adapter, asset_id)
|
||||
|
||||
|
||||
def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None:
|
||||
for clients in adapter.client_sets():
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
||||
@@ -40,7 +46,19 @@ def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
if info:
|
||||
return info
|
||||
except Exception as exc: # noqa: BLE001 — best-effort backfill, keep going
|
||||
log.info("meta extract failed for %s (%s): %s", url, clients, str(exc)[:100])
|
||||
# The asset id, not the URL. A source URL is a row of this user's download
|
||||
# history, and `redact_text` only takes credential params out of it — the rest
|
||||
# of it goes in whole. The id names the same asset to anyone holding the
|
||||
# database and nothing at all to anyone who is not; the host is kept because it
|
||||
# is what decides which client set was tried. yt-dlp's message goes through
|
||||
# `log_safe` for the same reason: it opens with the URL or the video id.
|
||||
log.info(
|
||||
"meta extract failed for asset %s on %s (%s): %s",
|
||||
asset_id,
|
||||
urlsplit(url).hostname or "?", # log-pii-ok: the host, never the path or query
|
||||
clients,
|
||||
formats.log_safe(exc)[:100],
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -60,12 +78,14 @@ def run() -> dict:
|
||||
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
||||
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||
info = _extract_meta(url, adapter)
|
||||
info = _extract_meta(url, adapter, a.id)
|
||||
uu = adapter.uploader_url(info) if info else None
|
||||
if uu:
|
||||
a.uploader_url = uu
|
||||
filled_url += 1
|
||||
log.info("asset %s uploader_url <- %s", a.id, uu)
|
||||
log.info(
|
||||
"asset %s uploader_url filled", a.id
|
||||
) # the URL names what they downloaded
|
||||
if a.poster_path is None:
|
||||
# Record a self-hosted poster for every download (the reliable og:image source):
|
||||
# ensure_poster returns the existing <base>.jpg thumbnail sidecar, or cuts a frame.
|
||||
@@ -73,7 +93,7 @@ def run() -> dict:
|
||||
if rel:
|
||||
a.poster_path = rel
|
||||
filled_poster += 1
|
||||
log.info("asset %s poster <- %s", a.id, rel)
|
||||
log.info("asset %s poster filled", a.id) # the path carries the title
|
||||
db.commit()
|
||||
result = {
|
||||
"assets": len(assets),
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Per-user, per-host download cookies — the single source of truth for "does the acting user's
|
||||
cookie apply to this download, and how does that keep the result private".
|
||||
|
||||
Two sides depend on ONE consistent answer (house rule: an invariant lives in shared code, not
|
||||
in prose):
|
||||
* `enqueue` salts the MediaAsset cache signature with the user id WHEN a cookie applies, so the
|
||||
authed download becomes a PRIVATE asset that is never cache-served to another user. Retry
|
||||
(`service.resume`) and Re-download (`service.redownload`) ask the same question again, so a
|
||||
cookie the user added after a failure goes with the next attempt.
|
||||
* the worker attaches the cookie file to yt-dlp ONLY when a cookie still applies AND the asset it
|
||||
is downloading into is that private, salted identity — so a cookie can never poison a shared
|
||||
asset (the load-bearing cross-user privacy fix).
|
||||
|
||||
Both key the private identity on `cookie_salt` / `host_key_for_url` here, so they can't drift.
|
||||
|
||||
Security controls enforced here:
|
||||
* **host scope (SEC-12)** — `host_key_for_url` matches the download URL's netloc against a curated
|
||||
registry via the netloc-scoped `_host_is`, so a cookie is only ever offered to the exact site it
|
||||
belongs to; an arbitrary `?url=` (or a look-alike host) resolves to no host_key and gets no
|
||||
cookie.
|
||||
* **secret handling** — cookies are Fernet-encrypted at rest; the blob is only ever decrypted
|
||||
in-worker into a 0600 file that is unlinked in a `finally`, and is never returned by a route,
|
||||
logged, or written to a job's error/snapshot.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.downloads import formats
|
||||
from app.downloads.adapters import _host_is
|
||||
from app.models import DownloadCredential, MediaAsset
|
||||
from app.security import decrypt_optional, encrypt
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class HostSpec:
|
||||
"""A supported cookie host: a short canonical `key` (stored on the credential + folded into the
|
||||
cache salt), a human `label` for the UI, and the netloc `domains` it matches."""
|
||||
|
||||
__slots__ = ("key", "label", "domains")
|
||||
|
||||
def __init__(self, key: str, label: str, domains: tuple[str, ...]):
|
||||
self.key = key
|
||||
self.label = label
|
||||
self.domains = domains
|
||||
|
||||
|
||||
# Curated registry (user's call: seed the common auth-walled hosts now). One `key` can cover several
|
||||
# domains of the same site. Add a host by adding a row — the FE dropdown, host-scope guard, and salt
|
||||
# all derive from this, so there is nothing else to wire up.
|
||||
SUPPORTED_HOSTS: tuple[HostSpec, ...] = (
|
||||
HostSpec("facebook", "Facebook", ("facebook.com", "fb.watch")),
|
||||
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
||||
# job 273: every player client fails on that one video while its neighbours download fine, and
|
||||
# the newest yt-dlp fails the same way — a signed-in session is the only way through). Without a
|
||||
# row here the `login_required` copy sent the user to a dropdown that could not help them.
|
||||
HostSpec("youtube", "YouTube", ("youtube.com", "youtu.be", "youtube-nocookie.com")),
|
||||
HostSpec("instagram", "Instagram", ("instagram.com",)),
|
||||
HostSpec("twitter", "X (Twitter)", ("twitter.com", "x.com")),
|
||||
HostSpec("tiktok", "TikTok", ("tiktok.com",)),
|
||||
HostSpec("reddit", "Reddit", ("reddit.com",)),
|
||||
)
|
||||
|
||||
_BY_KEY: dict[str, HostSpec] = {h.key: h for h in SUPPORTED_HOSTS}
|
||||
|
||||
|
||||
def is_supported_host(host_key: str) -> bool:
|
||||
return host_key in _BY_KEY
|
||||
|
||||
|
||||
def host_label(host_key: str) -> str:
|
||||
spec = _BY_KEY.get(host_key)
|
||||
return spec.label if spec else host_key
|
||||
|
||||
|
||||
def host_key_for_url(url: str | None) -> str | None:
|
||||
"""The supported host_key whose domains own `url`'s netloc, or None. Netloc-scoped (via
|
||||
`_host_is`), so a look-alike host ('myfacebook.com', 'facebook.com.evil') never matches. When
|
||||
several specs could match, the most-specific (longest matched domain) wins so the result is
|
||||
deterministic across the enqueue and worker call sites."""
|
||||
if not url:
|
||||
return None
|
||||
from urllib.parse import urlparse
|
||||
|
||||
host = urlparse(url).netloc.lower()
|
||||
if not host:
|
||||
return None
|
||||
best_key: str | None = None
|
||||
best_len = -1
|
||||
for spec in SUPPORTED_HOSTS:
|
||||
for dom in spec.domains:
|
||||
if _host_is(host, dom) and len(dom) > best_len:
|
||||
best_key, best_len = spec.key, len(dom)
|
||||
return best_key
|
||||
|
||||
|
||||
def cookie_salt(user_id: int, host_key: str) -> str:
|
||||
"""The cache-signature salt for a cookie-authed download. Keyed by user id + host so the authed
|
||||
asset is private to that user (never dedup-shared) and stable across that user's own re-adds."""
|
||||
return f"cookie:{user_id}:{host_key}"
|
||||
|
||||
|
||||
def authed_format_sig(spec: dict, user_id: int, host_key: str) -> str:
|
||||
"""The salted `format_sig` an authed download for (user, host) resolves to under the current
|
||||
signature version — the private asset enqueue creates, and one of the identities `is_authed_asset`
|
||||
recognises. Neither calls it; the tests use it to hold the two to each other."""
|
||||
return formats.format_sig(spec, salt=cookie_salt(user_id, host_key))
|
||||
|
||||
|
||||
def rebuild_salt(asset_id: int, cookie: str | None) -> str:
|
||||
"""The salt of a Re-download's fresh asset: its OWN row id (so every rebuild is a new identity no
|
||||
enqueue can ever compute), prefixed with the user's `cookie_salt` when a cookie applies. The id is
|
||||
what makes it recognisable — the worker has the asset row, so `is_authed_asset` can recompute this
|
||||
one, where a salt built from the job's previous asset id could not be."""
|
||||
return f"{cookie}:rebuild:{asset_id}" if cookie else f"rebuild:{asset_id}"
|
||||
|
||||
|
||||
def is_authed_asset(asset: MediaAsset, spec: dict, user_id: int, url: str | None) -> bool:
|
||||
"""Whether an asset is `user_id`'s private, cookie-salted identity for `url`'s host — the one
|
||||
kind of asset a cookie is ever attached to: the one enqueue creates (`authed_format_sig`), or a
|
||||
Re-download's rebuild of it (`rebuild_salt`). The worker's cookie gate asks it before a download,
|
||||
and the failure diagnosis asks it after one (a failure on this asset happened WITH the cookie).
|
||||
Either signature version counts (`formats.format_sigs`): a job queued before a deploy that bumped
|
||||
the version still downloads into the asset it was given."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return False
|
||||
salt = cookie_salt(user_id, host_key)
|
||||
if asset.format_sig in formats.format_sigs(spec, salt=salt):
|
||||
return True
|
||||
return asset.id is not None and asset.format_sig in formats.format_sigs(
|
||||
spec, salt=rebuild_salt(asset.id, salt)
|
||||
)
|
||||
|
||||
|
||||
def salt_for_enqueue(db: Session, user_id: int, url: str | None) -> str | None:
|
||||
"""At enqueue: the cache salt to apply if this user has a cookie for `url`'s host, else None (a
|
||||
normal shared download). Returns the salt string so the caller passes it straight to
|
||||
`format_sig(spec, salt=...)`."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return None
|
||||
if get_credential(db, user_id, host_key) is None:
|
||||
return None
|
||||
return cookie_salt(user_id, host_key)
|
||||
|
||||
|
||||
# --- storage (CRUD) ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def get_credential(db: Session, user_id: int, host_key: str) -> DownloadCredential | None:
|
||||
return db.execute(
|
||||
select(DownloadCredential).where(
|
||||
DownloadCredential.user_id == user_id, DownloadCredential.host_key == host_key
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
class InvalidCookies(ValueError):
|
||||
"""The pasted cookies aren't a usable Netscape cookies.txt — surfaced as a 400 by the route."""
|
||||
|
||||
|
||||
def validate_cookies(raw: str) -> str:
|
||||
"""Sanity-check a pasted cookies.txt (Netscape format) and return the cleaned text. yt-dlp reads
|
||||
the Netscape format: comment/`#` lines plus TAB-separated cookie lines (domain, flag, path,
|
||||
secure, expiry, name, value). We don't fully parse — just reject obvious non-cookie paste (empty,
|
||||
or no tab-separated data line) so a mistaken paste fails HERE with a clear message rather than
|
||||
silently producing a cookie file yt-dlp ignores."""
|
||||
text = (raw or "").strip()
|
||||
if not text:
|
||||
raise InvalidCookies("empty")
|
||||
has_data_line = any(
|
||||
line.strip() and not line.lstrip().startswith("#") and "\t" in line
|
||||
for line in text.splitlines()
|
||||
)
|
||||
if not has_data_line:
|
||||
raise InvalidCookies("format")
|
||||
# Netscape cookie files must end with a newline for some parsers; normalise.
|
||||
return text if text.endswith("\n") else text + "\n"
|
||||
|
||||
|
||||
def upsert_credential(db: Session, user_id: int, host_key: str, raw: str) -> DownloadCredential:
|
||||
"""Create or replace this user's cookie blob for `host_key`. Caller has validated host_key is
|
||||
supported and `raw` via `validate_cookies`. Does NOT commit (the route commits with the audit
|
||||
row so the trail can't drift). The blob is encrypted here; the plaintext never leaves this call."""
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
enc = encrypt(raw)
|
||||
if cred is None:
|
||||
cred = DownloadCredential(user_id=user_id, host_key=host_key, cookies_enc=enc)
|
||||
db.add(cred)
|
||||
else:
|
||||
cred.cookies_enc = enc
|
||||
return cred
|
||||
|
||||
|
||||
def delete_credential(db: Session, user_id: int, host_key: str) -> bool:
|
||||
"""Delete this user's cookie for `host_key`. Returns whether a row existed. Does NOT commit."""
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
if cred is None:
|
||||
return False
|
||||
db.delete(cred)
|
||||
return True
|
||||
|
||||
|
||||
def status_list(db: Session, user_id: int) -> list[dict]:
|
||||
"""The supported hosts with this user's configured/updated state — the write-only Settings view.
|
||||
Never includes the blob."""
|
||||
have = {
|
||||
c.host_key: c
|
||||
for c in db.execute(
|
||||
select(DownloadCredential).where(DownloadCredential.user_id == user_id)
|
||||
).scalars()
|
||||
}
|
||||
out = []
|
||||
for spec in SUPPORTED_HOSTS:
|
||||
cred = have.get(spec.key)
|
||||
out.append(
|
||||
{
|
||||
"host_key": spec.key,
|
||||
"label": spec.label,
|
||||
"configured": cred is not None,
|
||||
"updated_at": cred.updated_at.isoformat() if cred is not None else None,
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
# --- worker: resolve + write the cookie file --------------------------------------------------
|
||||
|
||||
|
||||
def usable_cookie(
|
||||
db: Session, user_id: int, url: str | None, asset: MediaAsset, spec: dict
|
||||
) -> str | None:
|
||||
"""The user's cookie text a download of `url` into `asset` goes with, or None — the ONE decision
|
||||
whether a download is signed in. The worker child writes it out for yt-dlp (`resolve_cookiefile`);
|
||||
the worker asks it again for a job that inherits a failed asset. That second ask sees the
|
||||
credential as it is when the job is claimed: a cookie the user removed or added since the failed
|
||||
download changes this job's explanation to fit the cookie its Retry would go with.
|
||||
|
||||
The double gate that prevents cross-user leakage:
|
||||
1. a credential must still exist for `url`'s host (host-scoped via `host_key_for_url`), AND
|
||||
2. the asset being downloaded into must BE this user's private, salted identity
|
||||
(`is_authed_asset`).
|
||||
|
||||
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
||||
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
||||
InvalidToken) also yields None; the download then fails honestly on the login wall (S3's
|
||||
`login_required`)."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return None
|
||||
cred = get_credential(db, user_id, host_key)
|
||||
if cred is None:
|
||||
return None
|
||||
if not is_authed_asset(asset, spec, user_id, url):
|
||||
return None # shared asset — never attach a cookie to a non-private identity
|
||||
raw = decrypt_optional(cred.cookies_enc)
|
||||
if not raw:
|
||||
log.warning(
|
||||
"download cookie for user %s host %s could not be decrypted (rotated key?) — "
|
||||
"treated as no cookie",
|
||||
user_id,
|
||||
host_key,
|
||||
)
|
||||
return None
|
||||
return raw
|
||||
|
||||
|
||||
def resolve_cookiefile(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
url: str | None,
|
||||
asset: MediaAsset,
|
||||
spec: dict,
|
||||
staging: Path,
|
||||
) -> Path | None:
|
||||
"""In the worker child: the cookies.txt to hand yt-dlp for this download (`usable_cookie`), or None."""
|
||||
raw = usable_cookie(db, user_id, url, asset, spec)
|
||||
return _write_cookiefile(raw, staging) if raw else None
|
||||
|
||||
|
||||
def _write_cookiefile(raw: str, staging: Path) -> Path:
|
||||
"""Write the cookie blob to a private 0600 file under `staging/.auth/` and return its path.
|
||||
|
||||
Deliberately NOT in the staging root: the download retry loop unlinks every file it finds there,
|
||||
and `_produced_file` picks the largest file there as the output — either would corrupt a
|
||||
cookies.txt in the root. A `.auth` SUBDIR survives both (they skip directories) and is still
|
||||
removed by the parent worker's staging cleanup on a hard kill. The caller unlinks it in a
|
||||
`finally` on the normal/exception paths."""
|
||||
auth_dir = staging / ".auth"
|
||||
auth_dir.mkdir(mode=0o700, exist_ok=True)
|
||||
path = auth_dir / "cookies.txt"
|
||||
# Open with 0600 from the start (don't create world-readable then chmod).
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
try:
|
||||
os.write(fd, raw.encode("utf-8"))
|
||||
finally:
|
||||
os.close(fd)
|
||||
return path
|
||||
+143
-46
@@ -23,8 +23,8 @@ for a player hover-preview later.
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -32,7 +32,21 @@ import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("siftlode.edit")
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.downloads.stated import StatedFailure
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def encode_thread_cap() -> int:
|
||||
"""How many encoder threads a browser-safe re-encode may use. `download_ffmpeg_threads` when the
|
||||
admin pinned one (>0), else reserve a core for foreground work (max(1, cpu_count - 1)). Shared by
|
||||
the ffmpeg `-threads` flag and the child's `OMP_NUM_THREADS` so both honour the same budget."""
|
||||
n = settings.download_ffmpeg_threads
|
||||
if n > 0:
|
||||
return n
|
||||
return max(1, (os.cpu_count() or 2) - 1)
|
||||
|
||||
|
||||
class EditAborted(Exception):
|
||||
@@ -41,6 +55,10 @@ class EditAborted(Exception):
|
||||
|
||||
# --- edit spec ------------------------------------------------------------------------------
|
||||
|
||||
# SEC-13: ceiling on a join's cut-list. An accurate join emits two filters per segment into a single
|
||||
# ffmpeg `filter_complex`, so "however many the caller sent" is an unbounded command line.
|
||||
_MAX_SEGMENTS = 100
|
||||
|
||||
|
||||
def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
"""Canonicalize a raw edit spec: clamp/round trim, coerce crop to ints, drop empties.
|
||||
@@ -54,7 +72,13 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
if crop:
|
||||
try:
|
||||
c = {k: int(round(float(crop[k]))) for k in ("x", "y", "w", "h")}
|
||||
except (KeyError, TypeError, ValueError):
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
# The clip still renders — uncropped. The user asked for a crop, got a whole frame,
|
||||
# and nothing anywhere called it a failure; this line is the only trace there is.
|
||||
log.warning(
|
||||
"the crop in an edit spec could not be read and was dropped",
|
||||
extra=logs.event("edit.spec.crop_dropped", reason=type(exc).__name__),
|
||||
)
|
||||
c = None
|
||||
if c and c["w"] > 0 and c["h"] > 0 and c["x"] >= 0 and c["y"] >= 0:
|
||||
out["crop"] = c
|
||||
@@ -63,6 +87,7 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
segs = spec.get("segments")
|
||||
if isinstance(segs, list) and segs:
|
||||
norm: list[dict] = []
|
||||
unreadable = 0
|
||||
for s in segs:
|
||||
try:
|
||||
st = max(0.0, float((s or {}).get("start_s") or 0.0))
|
||||
@@ -71,10 +96,25 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
continue
|
||||
en = float(end_raw)
|
||||
except (TypeError, ValueError):
|
||||
# silent-ok: counted, not logged here — one cut-list can drop several, and the
|
||||
# join that comes out is simply shorter than the one the user built. Reported
|
||||
# once after the loop.
|
||||
unreadable += 1
|
||||
continue
|
||||
if en > st:
|
||||
norm.append({"start_s": round(st, 3), "end_s": round(en, 3)})
|
||||
if unreadable:
|
||||
log.warning(
|
||||
"some segments of a cut list could not be read and were dropped",
|
||||
extra=logs.event("edit.spec.segments_dropped", dropped=unreadable, kept=len(norm)),
|
||||
)
|
||||
norm.sort(key=lambda x: x["start_s"])
|
||||
# SEC-13: an accurate join builds one ffmpeg `filter_complex` with two filters per segment,
|
||||
# so an unbounded list becomes an unbounded command line and an ffmpeg that never finishes.
|
||||
# The cap is far above any real cut-list (the UI's own splitter produces a handful) and the
|
||||
# excess is DROPPED rather than raising: the surplus segments are the attack, the first
|
||||
# _MAX_SEGMENTS are a legitimate edit.
|
||||
del norm[_MAX_SEGMENTS:]
|
||||
if norm:
|
||||
out["segments"] = norm
|
||||
# A join always chooses a codec path: accurate=filter-concat re-encode,
|
||||
@@ -91,7 +131,13 @@ def normalize_edit_spec(spec: dict | None) -> dict:
|
||||
start = max(0.0, float(trim.get("start_s") or 0.0))
|
||||
end_raw = trim.get("end_s")
|
||||
end = float(end_raw) if end_raw is not None else None
|
||||
except (TypeError, ValueError):
|
||||
except (TypeError, ValueError) as exc:
|
||||
# Either a 400 the user cannot act on, or — beside a crop — a full-length clip where
|
||||
# they asked for a cut. Both are worth one line saying the spec was the problem.
|
||||
log.warning(
|
||||
"the trim in an edit spec could not be read and was dropped",
|
||||
extra=logs.event("edit.spec.trim_dropped", reason=type(exc).__name__),
|
||||
)
|
||||
start, end = 0.0, None
|
||||
t: dict = {"start_s": round(start, 3)}
|
||||
if end is not None and end > start:
|
||||
@@ -283,11 +329,14 @@ def _parse_out_time(line: str) -> float | None:
|
||||
if line.startswith("out_time_us="):
|
||||
try:
|
||||
return int(line.split("=", 1)[1]) / 1_000_000
|
||||
# silent-ok: one line of ffmpeg's `-progress` stream, several per second. None means "this
|
||||
# line carried no usable time", which the caller already treats as "no progress yet".
|
||||
except ValueError:
|
||||
return None
|
||||
if line.startswith("out_time_ms="): # (some builds mislabel this as microseconds)
|
||||
try:
|
||||
return int(line.split("=", 1)[1]) / 1_000_000
|
||||
# silent-ok: as above, the other spelling of the same line.
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
@@ -306,7 +355,9 @@ def _pump(stream, sink: "queue.Queue[str | None]") -> None:
|
||||
try:
|
||||
for line in stream:
|
||||
sink.put(line)
|
||||
except (ValueError, OSError): # pipe closed under us by the kill path
|
||||
# silent-ok: the pipe was closed under us by our OWN kill path — the expected end of a
|
||||
# cancelled or timed-out run, and the code that did the killing is the code that reports it.
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
finally:
|
||||
sink.put(None)
|
||||
@@ -324,6 +375,8 @@ def _drain_tail(stream, sink: list[str]) -> None:
|
||||
sink.append(line)
|
||||
if len(sink) > 200:
|
||||
del sink[:100]
|
||||
# silent-ok: same closed pipe as `_pump` above, and the tail collected so far is still handed
|
||||
# to whoever reports the failure.
|
||||
except (ValueError, OSError):
|
||||
pass
|
||||
|
||||
@@ -339,13 +392,17 @@ def escalate_kill(proc: subprocess.Popen, terminate, kill, grace_s: float, label
|
||||
try:
|
||||
proc.wait(timeout=grace_s)
|
||||
return
|
||||
# silent-ok: the escalation below IS the handling of this timeout, and surviving THAT is what
|
||||
# the line under the second wait reports.
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
kill()
|
||||
try:
|
||||
proc.wait(timeout=grace_s)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid)
|
||||
# noqa TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
||||
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
||||
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
||||
|
||||
|
||||
def _stop(proc: subprocess.Popen) -> None:
|
||||
@@ -381,6 +438,8 @@ def run_ffmpeg(
|
||||
while True:
|
||||
try:
|
||||
line = lines.get(timeout=1.0)
|
||||
# silent-ok: not a failure — the one-second timeout is how this loop gets a turn to
|
||||
# poll for cancellation while ffmpeg is quiet.
|
||||
except queue.Empty:
|
||||
line = ""
|
||||
if line is None: # stdout hit EOF — ffmpeg is done (or dying)
|
||||
@@ -393,11 +452,11 @@ def run_ffmpeg(
|
||||
if should_cancel():
|
||||
raise EditAborted
|
||||
if time.monotonic() - last_output > stall_timeout_s:
|
||||
raise RuntimeError(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
|
||||
raise StatedFailure(f"ffmpeg stalled: no progress for {int(stall_timeout_s)}s")
|
||||
try:
|
||||
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
|
||||
except subprocess.TimeoutExpired: # pipes closed but the process lingers
|
||||
raise RuntimeError("ffmpeg did not exit after closing its output")
|
||||
raise StatedFailure("ffmpeg did not exit after closing its output")
|
||||
except BaseException:
|
||||
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
|
||||
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
|
||||
@@ -412,7 +471,7 @@ def run_ffmpeg(
|
||||
if pipe:
|
||||
pipe.close()
|
||||
if ret != 0:
|
||||
raise RuntimeError(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
raise StatedFailure(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
|
||||
|
||||
# --- ffmpeg: universal browser playability --------------------------------------------------
|
||||
@@ -425,18 +484,48 @@ _BROWSER_SAFE_VCODEC = ("h264",)
|
||||
_BROWSER_SAFE_ACODEC = ("aac", "mp3")
|
||||
|
||||
|
||||
def parse_probe_streams(payload: str) -> tuple[str | None, str | None]:
|
||||
"""(video_codec, audio_codec) out of ffprobe's JSON stream list, lowercased.
|
||||
|
||||
JSON, not the flat `default=nw=1` output: there the fields arrive in ffprobe's own struct
|
||||
order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line "remember the last
|
||||
codec_type" parser attributes every name to the PREVIOUS stream. Measured on real downloads:
|
||||
an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
|
||||
believed the video stream was AAC and re-encoded a perfectly fine file on every download."""
|
||||
def run_ffprobe_json(
|
||||
path: Path,
|
||||
*,
|
||||
show_streams: bool = False,
|
||||
show_format: bool = False,
|
||||
entries: str | None = None,
|
||||
) -> dict | None:
|
||||
"""One ffprobe call → its parsed JSON dict, or None on any failure (timeout, non-zero exit,
|
||||
missing binary, unparseable output). The SINGLE ffprobe-JSON-invocation point in the downloads
|
||||
package: the browser-playability codec probe here and the shared media probe (`probe.probe_media`,
|
||||
used by both the download worker's finalize and the upload ingest) both go through it, so the
|
||||
subprocess flags/timeout/error handling live in one place instead of drifting copies."""
|
||||
args = ["ffprobe", "-v", "error"]
|
||||
if show_streams:
|
||||
args.append("-show_streams")
|
||||
if show_format:
|
||||
args.append("-show_format")
|
||||
if entries:
|
||||
args += ["-show_entries", entries]
|
||||
args += ["-of", "json", str(path)]
|
||||
try:
|
||||
streams = json.loads(payload or "{}").get("streams") or []
|
||||
except ValueError:
|
||||
return None, None
|
||||
out = subprocess.run(args, capture_output=True, text=True, timeout=30, check=True).stdout
|
||||
return json.loads(out)
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError) as exc:
|
||||
# None makes every field it would have filled read as absent, and a file with no streams
|
||||
# and no duration is indistinguishable here from one that is not media at all.
|
||||
log.warning(
|
||||
"ffprobe could not read a file",
|
||||
extra=logs.event("media.ffprobe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _codecs_from_streams(streams: list) -> tuple[str | None, str | None]:
|
||||
"""(video_codec, audio_codec) lowercased from an ffprobe JSON stream list — the first of each kind.
|
||||
|
||||
Parsed from the JSON `streams` list, NOT ffprobe's flat `default=nw=1` output: there the fields
|
||||
arrive in ffprobe's own struct order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line
|
||||
"remember the last codec_type" parser attributes every name to the PREVIOUS stream. Measured on real
|
||||
downloads: an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
|
||||
believed the video stream was AAC and re-encoded a perfectly fine file on every download. Reading each
|
||||
stream dict by its own `codec_type` is immune to that ordering."""
|
||||
vcodec = acodec = None
|
||||
for s in streams:
|
||||
name = (s.get("codec_name") or "").lower() or None
|
||||
@@ -451,26 +540,10 @@ def parse_probe_streams(payload: str) -> tuple[str | None, str | None]:
|
||||
def probe_codecs(path: Path) -> tuple[str | None, str | None]:
|
||||
"""Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing
|
||||
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"stream=codec_type,codec_name",
|
||||
"-of",
|
||||
"json",
|
||||
str(path),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
check=True,
|
||||
).stdout
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError):
|
||||
data = run_ffprobe_json(path, entries="stream=codec_type,codec_name")
|
||||
if data is None:
|
||||
return None, None
|
||||
return parse_probe_streams(out)
|
||||
return _codecs_from_streams(data.get("streams") or [])
|
||||
|
||||
|
||||
def probe_duration(path: Path) -> int:
|
||||
@@ -494,7 +567,13 @@ def probe_duration(path: Path) -> int:
|
||||
check=True,
|
||||
).stdout.strip()
|
||||
return int(float(out)) if out else 0
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError):
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError, ValueError) as exc:
|
||||
# Zero is the denominator of the re-encode progress bar, so a failed probe leaves the user
|
||||
# watching a bar that never moves rather than a job that says it went wrong.
|
||||
log.info(
|
||||
"the duration probe failed; progress for this job has no total",
|
||||
extra=logs.event("media.duration_probe_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
@@ -510,11 +589,12 @@ def browser_transcode_flags(
|
||||
|
||||
|
||||
def build_browser_safe_cmd(
|
||||
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool
|
||||
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool, threads: int | None = None
|
||||
) -> list[str]:
|
||||
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
||||
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
||||
pipe:1` streams machine-readable progress for run_ffmpeg."""
|
||||
pipe:1` streams machine-readable progress for run_ffmpeg. `threads` caps the encoder so a
|
||||
re-encode can't claim every core on the shared box (see the worker-child de-prioritization)."""
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
@@ -534,6 +614,8 @@ def build_browser_safe_cmd(
|
||||
else ["-c:v", "copy"]
|
||||
)
|
||||
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
||||
if threads and threads > 0:
|
||||
cmd += ["-threads", str(threads)]
|
||||
cmd += ["-movflags", "+faststart", str(dest)]
|
||||
return cmd
|
||||
|
||||
@@ -636,15 +718,28 @@ def _run_ffmpeg_best_effort(cmd: list[str], timeout_s: int) -> Exception | None:
|
||||
try:
|
||||
subprocess.run(cmd, capture_output=True, timeout=timeout_s, check=True)
|
||||
return None
|
||||
# silent-ok: the exception is the RETURN VALUE. This helper exists so the three callers share
|
||||
# one run/timeout policy, and each logs the failure with the context only it has.
|
||||
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, OSError) as exc:
|
||||
return exc
|
||||
|
||||
|
||||
def _step_failure(err: BaseException) -> str:
|
||||
"""What a best-effort image step's failure line says: the kind and, for ffmpeg, its exit
|
||||
status. Never `str(err)`: a CalledProcessError or TimeoutExpired quotes ffmpeg's argv, and an
|
||||
OSError its file name — the source path either way, which is built from the channel and the
|
||||
title."""
|
||||
code = getattr(err, "returncode", None)
|
||||
return f"{type(err).__name__} {code}" if code is not None else type(err).__name__
|
||||
|
||||
|
||||
def _is_jpeg(path: Path) -> bool:
|
||||
"""True if the file starts with the JPEG SOI marker (FF D8 FF)."""
|
||||
try:
|
||||
with path.open("rb") as f:
|
||||
return f.read(3) == b"\xff\xd8\xff"
|
||||
# silent-ok: False only means "do not copy this straight through" — the caller then goes the
|
||||
# ffmpeg route, which opens the same file and reports it properly if it really is unreadable.
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
@@ -665,13 +760,15 @@ def thumbnail_to_jpg(src: Path | None, dest: Path, timeout_s: int = 30) -> Path
|
||||
shutil.copyfile(src, dest)
|
||||
return dest
|
||||
except OSError as exc:
|
||||
log.info("thumbnail copy failed (%s); poster falls back to a cut frame", str(exc)[:120])
|
||||
log.info(
|
||||
"thumbnail copy failed (%s); poster falls back to a cut frame", _step_failure(exc)
|
||||
)
|
||||
return None
|
||||
err = _run_ffmpeg_best_effort(build_thumbnail_jpg_cmd(src, dest), timeout_s)
|
||||
if err is not None:
|
||||
log.info(
|
||||
"thumbnail->jpg conversion failed (%s); poster falls back to a cut frame",
|
||||
str(err)[:120],
|
||||
_step_failure(err),
|
||||
)
|
||||
return None
|
||||
return dest if dest.exists() else None
|
||||
@@ -696,7 +793,7 @@ def ensure_poster(asset, download_root: str, timeout_s: int = 30) -> str | None:
|
||||
at = min(3.0, (asset.duration_s or 10) * 0.1)
|
||||
err = _run_ffmpeg_best_effort(build_poster_cmd(src, dest, at), timeout_s)
|
||||
if err is not None:
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, str(err)[:120])
|
||||
log.info("poster gen skipped for asset %s: %s", asset.id, _step_failure(err))
|
||||
return None
|
||||
return rel if dest.exists() else None
|
||||
|
||||
@@ -727,7 +824,7 @@ def ensure_storyboard(asset, download_root: str, timeout_s: int = 90) -> dict |
|
||||
cmd = build_storyboard_cmd(src, dest, asset.duration_s, geom)
|
||||
err = _run_ffmpeg_best_effort(cmd, timeout_s)
|
||||
if err is not None:
|
||||
log.info("storyboard gen skipped for asset %s: %s", asset.id, str(err)[:120])
|
||||
log.info("storyboard gen skipped for asset %s: %s", asset.id, _step_failure(err))
|
||||
return None
|
||||
if not dest.exists():
|
||||
return None
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code.
|
||||
"""Classify a raw yt-dlp / ffmpeg failure string into a stable category code — and, where a site's
|
||||
failure has been measured, into that site's entry of the failure catalog.
|
||||
|
||||
The worker stores the raw exception text (truncated) for technical detail, but a raw
|
||||
`ERROR: [youtube] <id>: Private video. Sign in…` is noise to a user. `classify()` maps that text
|
||||
@@ -6,11 +7,66 @@ to one of a small set of category codes; the frontend turns the code into a loca
|
||||
explanation (and decides whether a retry makes sense). The raw text stays available behind a
|
||||
"details" disclosure. This is the same string-matching approach the worker already trusts for
|
||||
`_RETRIABLE_MARKERS` — kept here as the single source of truth for "what kind of failure is this".
|
||||
|
||||
**The failure catalog** (`_CATALOG`, E11.5). A category is generic: "Sign-in required" cannot tell a
|
||||
Facebook user that the cookie they already gave was refused, and yt-dlp words several different
|
||||
Facebook walls identically (`Cannot parse data`). A catalog entry is scoped to ONE host and may also
|
||||
depend on whether a cookie was attached and on the kind of link, and names a `detail` code the UI has
|
||||
its own sentence for — why it failed on that site, and what the user can do. `diagnose()` tries the
|
||||
catalog first and falls back to the category. How an entry is added, every time a failure is reported:
|
||||
1. reproduce it on dev (with and without the user's cookie) and keep the verbatim yt-dlp text;
|
||||
2. add the entry below, scoped as narrowly as the measurement allows (a marker that is yt-dlp's
|
||||
generic wording for every site must never match without its host; a marker another module also
|
||||
decides on — the Facebook Chrome retry's FACEBOOK_PARSE_WALL — is imported, never retyped);
|
||||
3. pin the real text in `tests/test_download_errors.py`;
|
||||
4. add `downloads.errors.fail.catalog.<detail>.{short,body}` in EN and HU (a contract test checks it;
|
||||
the group is not called `detail`, which `actionErrors.contract` reads as a raw `HttpError.detail`).
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import NamedTuple
|
||||
|
||||
from app.downloads.adapters import (
|
||||
FACEBOOK_PARSE_WALL,
|
||||
POST_NOT_SHOWN_MESSAGE,
|
||||
POST_WITHOUT_VIDEO_MESSAGE,
|
||||
facebook_group_post_id,
|
||||
)
|
||||
from app.downloads.credentials import host_key_for_url
|
||||
|
||||
# Worker-authored strings that this module owns because each is coupled to `_RULES`.
|
||||
#
|
||||
# QUOTA_FULL_MESSAGE is raised by the worker and must classify back to `quota_exceeded`; its rule
|
||||
# below matches a fragment of it, and test_download_errors pins the pair, because a message and its
|
||||
# rule drifting apart is exactly how a category silently becomes unreachable.
|
||||
#
|
||||
# NETWORK_REDACTED_MESSAGE REPLACES a message after classification (`user_error_text`). It is written
|
||||
# to the shared asset too, and a second job that inherits that failed asset re-diagnoses the stored
|
||||
# text (`worker._process_job`), so it has a rule of its own that gives the category back.
|
||||
#
|
||||
# Other worker-raised texts are still inline literals at their raise sites. The ffmpeg ones and
|
||||
# "produced no output file" have markers here; the urlguard refusal deliberately has none and
|
||||
# classifies as `unknown` — it is a refusal, not a diagnosis, and no category would say more.
|
||||
#
|
||||
# A source adapter's verdicts on a link (`SourceAdapter.resolve_url`, E11.5 S4) are adapters.py's
|
||||
# constants, imported above; each has a rule below that gives its category back, and a catalog entry
|
||||
# that gives the site's sentence.
|
||||
QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
|
||||
NETWORK_REDACTED_MESSAGE = "The server could not reach that address."
|
||||
|
||||
# (marker, category) — first match wins, so order most-specific → most-general. Matching is
|
||||
# case-insensitive on a substring, against the raw yt-dlp/ffmpeg message.
|
||||
_RULES: tuple[tuple[str, str], ...] = (
|
||||
# Our own refusal: the worker found the user already at their storage cap (SEC-13). Placed
|
||||
# first — it is our wording, so it can never be a false match on an upstream message.
|
||||
("download storage is full", "quota_exceeded"),
|
||||
# Our own redaction of a transport failure (see NETWORK_REDACTED_MESSAGE) — our wording too.
|
||||
(NETWORK_REDACTED_MESSAGE.lower(), "network"),
|
||||
# Our own verdicts on a post link (adapters.py) — our wording too: its page holds no single video,
|
||||
# or the site answered the signed-in read with something that is not the post at all.
|
||||
(POST_WITHOUT_VIDEO_MESSAGE.lower(), "format_unavailable"),
|
||||
(POST_NOT_SHOWN_MESSAGE.lower(), "login_required"),
|
||||
# Login / age / members / bot-check — the user must be signed in (we deliberately are not).
|
||||
("sign in to confirm your age", "login_required"),
|
||||
("confirm your age", "login_required"),
|
||||
@@ -24,6 +80,16 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
# Some extractors (e.g. Reddit) gate content behind a logged-in account: "Account
|
||||
# authentication is required." — an auth wall we deliberately don't cross, not a transient error.
|
||||
("authentication is required", "login_required"),
|
||||
# Closed-group / private-post auth walls (Facebook groups, Instagram private, …). yt-dlp phrases
|
||||
# these many ways; the common thread is "you must be signed in" or its own suggestion to supply
|
||||
# cookies. Treating the cookie suggestion as an auth signal is what makes a login-walled link read
|
||||
# honestly (and stay non-retryable) when the acting user has no usable cookie for that host.
|
||||
("log in to", "login_required"),
|
||||
("login required", "login_required"),
|
||||
("you must be logged in", "login_required"),
|
||||
("only available for registered users", "login_required"),
|
||||
("use --cookies", "login_required"),
|
||||
("--cookies-from-browser", "login_required"),
|
||||
# Geo-restricted.
|
||||
("in your country", "geo_blocked"),
|
||||
("available in your location", "geo_blocked"),
|
||||
@@ -42,6 +108,10 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
("account associated with this video has been terminated", "unavailable"),
|
||||
("this video is not available", "unavailable"),
|
||||
("this video has been removed", "unavailable"),
|
||||
# yt-dlp doesn't support this site at all ("Unsupported URL: …") — distinct from an auth wall on a
|
||||
# supported site. Non-retryable: no cookie or retry can make an unsupported host downloadable.
|
||||
("unsupported url", "unsupported_host"),
|
||||
("no video could be found in this url", "unsupported_host"),
|
||||
# DRM.
|
||||
("drm protected", "drm"),
|
||||
("drm-protected", "drm"),
|
||||
@@ -68,6 +138,120 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
)
|
||||
|
||||
|
||||
class Diagnosis(NamedTuple):
|
||||
"""What a failure was: the generic `code` (category) and, when a catalog entry matched, its
|
||||
host-specific `detail`."""
|
||||
|
||||
code: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _Entry:
|
||||
"""One measured failure of one site. It matches when the download URL belongs to `host`
|
||||
(`credentials.SUPPORTED_HOSTS`), every one of `markers` is in the message, the cookie state is
|
||||
`cookie` (None: either), and — when `link` is given — it recognises the URL (a truthy result). A
|
||||
link predicate is the adapter's own, so the catalog and the download agree on what a link is."""
|
||||
|
||||
host: str
|
||||
markers: tuple[str, ...]
|
||||
code: str
|
||||
detail: str
|
||||
cookie: bool | None = None
|
||||
link: Callable[[str], object] | None = None
|
||||
|
||||
|
||||
_CATALOG: tuple[_Entry, ...] = (
|
||||
# Facebook answers a signed-out client with a page yt-dlp cannot parse — an 18+ reel, a closed
|
||||
# group's post (measured 2026-09-13: prod jobs 145, 156, 256 download once the owner's cookie is
|
||||
# attached). "Cannot parse data" is yt-dlp's generic wording for a broken extractor on EVERY site,
|
||||
# which is why the entry is scoped to Facebook and never becomes a general rule.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
||||
code="login_required",
|
||||
detail="facebook.login_wall",
|
||||
cookie=False,
|
||||
),
|
||||
# The same text WITH a cookie: an expired / signed-out cookie, or a page that also needs
|
||||
# impersonation (prod job 257). Telling this user to add cookies would be wrong — they have.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", FACEBOOK_PARSE_WALL),
|
||||
code="login_required",
|
||||
detail="facebook.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# A group post read with a cookie that Facebook answered with its login page (measured with a
|
||||
# refused cookie: no post id anywhere on the page) — the same refused cookie, found by the adapter.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=(POST_NOT_SHOWN_MESSAGE.lower(),),
|
||||
code="login_required",
|
||||
detail="facebook.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# A group post with no single video of its own. With a cookie the adapter reads the post's page
|
||||
# and says so itself (POST_WITHOUT_VIDEO_MESSAGE); without one yt-dlp is given the post link and
|
||||
# answers "No video formats found" (measured on prod jobs 66 and 145, whose videos are reshared
|
||||
# clips). With a cookie yt-dlp only ever gets the post's VIDEO link, so the same text there is about
|
||||
# that video, not the post — the category says it. Only for group post links (the adapter's own
|
||||
# test of one): a reel with no formats is a different failure.
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=(POST_WITHOUT_VIDEO_MESSAGE.lower(),),
|
||||
code="format_unavailable",
|
||||
detail="facebook.group_post_video",
|
||||
link=facebook_group_post_id,
|
||||
),
|
||||
_Entry(
|
||||
host="facebook",
|
||||
markers=("[facebook]", "no video formats found"),
|
||||
code="format_unavailable",
|
||||
detail="facebook.group_post_video",
|
||||
cookie=False,
|
||||
link=facebook_group_post_id,
|
||||
),
|
||||
# YouTube gates INDIVIDUAL videos behind "Sign in to confirm you're not a bot" (measured on prod
|
||||
# job 273, a festival set: all 12 of yt-dlp's player clients fail on that one video while its
|
||||
# neighbours download fine, the POT sidecar is live and answering, and the newest yt-dlp fails
|
||||
# identically — so this is the video's own signed-out gate, not a broken stack or a stale yt-dlp).
|
||||
# The marker is quote-agnostic on purpose: YouTube writes the apostrophe as U+2019, so
|
||||
# "you're not a bot" with a straight quote never matches the real text.
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("[youtube]", "not a bot"),
|
||||
code="login_required",
|
||||
detail="youtube.bot_wall",
|
||||
cookie=False,
|
||||
),
|
||||
# The same wall WITH a cookie. Not yet reproduced with a real YouTube cookie (this user's first
|
||||
# one is being added now) — but "add cookies" is provably the wrong advice for someone who has,
|
||||
# and yt-dlp names the likeliest cause itself: "The provided YouTube account cookies are no
|
||||
# longer valid. They have likely been rotated in the browser as a security measure."
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("[youtube]", "not a bot"),
|
||||
code="login_required",
|
||||
detail="youtube.cookie_rejected",
|
||||
cookie=True,
|
||||
),
|
||||
# The same gate one step later: the player response comes back, and then the CDN refuses the
|
||||
# stream itself ("unable to download video data: HTTP Error 403"). Measured 2026-09-16 on the
|
||||
# very same video, ~90 minutes after it was answering with the bot message — signed out it
|
||||
# 403'd on every attempt; with the owner's cookie attached it downloaded in full (519 MB). So
|
||||
# the category this text carries generically, `network`, is a lie here: nothing was wrong with
|
||||
# the network. Scoped to a cookie-less YouTube download, where that is what we measured.
|
||||
_Entry(
|
||||
host="youtube",
|
||||
markers=("unable to download video data", "403"),
|
||||
code="login_required",
|
||||
detail="youtube.media_forbidden",
|
||||
cookie=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def classify(msg: str | None) -> str:
|
||||
"""Return a category code for a raw failure message (``"unknown"`` if nothing matches)."""
|
||||
if not msg:
|
||||
@@ -77,3 +261,41 @@ def classify(msg: str | None) -> str:
|
||||
if marker in low:
|
||||
return category
|
||||
return "unknown"
|
||||
|
||||
|
||||
def diagnose(
|
||||
msg: str | None, *, url: str | None = None, cookie_attached: bool = False
|
||||
) -> Diagnosis:
|
||||
"""The catalog entry for this failure of this link, else just its category (see the module doc).
|
||||
|
||||
`url` is the address yt-dlp was given; `cookie_attached` whether the acting user's cookie went
|
||||
with it. Without a URL only the category can be known."""
|
||||
host = host_key_for_url(url) if msg and url else None
|
||||
if host is not None:
|
||||
low = msg.lower() # type: ignore[union-attr] — `host` is only set when `msg` is truthy
|
||||
for e in _CATALOG:
|
||||
if (
|
||||
e.host == host
|
||||
and all(m in low for m in e.markers)
|
||||
and (e.cookie is None or e.cookie == cookie_attached)
|
||||
and (e.link is None or e.link(url)) # type: ignore[arg-type] — as `msg` above
|
||||
):
|
||||
return Diagnosis(e.code, e.detail)
|
||||
return Diagnosis(classify(msg))
|
||||
|
||||
|
||||
def user_error_text(source_kind: str, msg: str, code: str) -> str:
|
||||
"""The failure text to STORE on the job, given the raw message and its category.
|
||||
|
||||
SEC-12: for a user-supplied URL, a transport-level failure text distinguishes "connection
|
||||
refused" from "HTTP Error 401" from "timed out" per host:port — which is a port scanner with
|
||||
extra steps, since the raw text is shown to the user behind the details disclosure. `urlguard`
|
||||
stops the fetch aiming at our own network; this stops the remaining readout.
|
||||
|
||||
Only the network class, and only for ``url`` jobs: a YouTube job has no user-chosen host, and
|
||||
every other class (login_required, geo_blocked, drm, …) is real, useful diagnosis. A separate
|
||||
function rather than an inline branch in the worker so it can be pinned by a test — the raw
|
||||
text leaking back is a security regression that would otherwise pass a green suite."""
|
||||
if source_kind == "url" and code == "network":
|
||||
return NETWORK_REDACTED_MESSAGE
|
||||
return msg
|
||||
|
||||
@@ -5,17 +5,146 @@ of truth for what file gets produced. `format_sig` hashes exactly the fields tha
|
||||
OUTPUT BYTES, so two users picking equivalent settings share one cached MediaAsset.
|
||||
`build_ydl_opts` turns a spec into a yt-dlp options dict (format selector + postprocessors).
|
||||
|
||||
spec shape (all keys optional; defaults applied by `normalize`):
|
||||
spec shape (all keys optional; defaults + browser-playability clamp applied by `normalize`):
|
||||
mode "av" (audio+video) | "v" (video-only) | "a" (audio-only)
|
||||
max_height int | null (null = best)
|
||||
container "mp4" | "mkv" | "webm" | null (merge/remux target for av/v)
|
||||
audio_format "m4a" | "mp3" | "opus" | null (extract target for mode "a")
|
||||
vcodec "h264" | "vp9" | "av1" | null (preference, best-effort)
|
||||
container always coerced to "mp4" (the only universally browser-playable container)
|
||||
audio_format "m4a" | "mp3" (mode "a" extract target; anything else — e.g. opus — → "m4a")
|
||||
vcodec always coerced to null → H.264 selected (see `normalize`); no opt-out
|
||||
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
|
||||
|
||||
`normalize` clamps container/vcodec/audio_format so EVERY download plays in-browser (E7); the
|
||||
`container`/`vcodec` keys survive only so an old stored profile deserializes without error.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import urllib.parse
|
||||
from collections.abc import Iterator
|
||||
|
||||
from app import logs
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# yt-dlp names what it is working on in these shapes, and each one is a row of someone's download
|
||||
# history: the source URL, the `[extractor] <id>:` prefix on nearly every warning and error, the
|
||||
# file path it writes to (`storage.rel_path` builds it from the channel and the title), a
|
||||
# playlist's title, and an id quoted mid-sentence. The job id is already on every line
|
||||
# (`logs.set_job` in the download child), so none of them adds anything an operator needs. What
|
||||
# stays: the host (it says which extractor is failing) and the extractor tag. Every pattern below
|
||||
# is a shape measured in yt-dlp's own source (2026.07.04), pinned in test_ytdlp_logging.py.
|
||||
_URL_RE = re.compile(r"https?://([^/\s]+)\S*")
|
||||
# `Destination:` is yt-dlp's own word in `[download] Destination: <path>`, not an id — the path
|
||||
# after it is what `_PATH_RE` takes out.
|
||||
_ID_PREFIX_RE = re.compile(r"(\[[\w:.-]+\] )(?!Destination:)[^\s:\[\]]+:")
|
||||
# The generic extractor's "id" is the URL's basename, percent-DECODED: `My%20Holiday.mp4` is
|
||||
# `My Holiday` — spaces, brackets, colons and all, which no pattern can bound (`Our Trip: Day 1`
|
||||
# ends at which `: `?). The job knows its own URL, so the exact name is taken out as a literal
|
||||
# (`hiding`). This pattern is the fallback for a line logged outside that scope: up to the first
|
||||
# `: `. Only this extractor: every other one names a site id, and letting the prefix run to `: `
|
||||
# everywhere would take the words out of lines like `[VideoConvertor] Converting …; Destination:`.
|
||||
_GENERIC_ID_RE = re.compile(r"(\[generic\] )(?!Destination:)[^\[\]\n]+?:(?= |$)", re.M)
|
||||
# `[youtube:tab] Playlist <title>: Downloading 5 items` — the title is not an id, and has spaces.
|
||||
_PLAYLIST_TITLE_RE = re.compile(r"(\] Playlist ).+?(?=: Downloading )")
|
||||
# `[download] Finished downloading playlist: <title>`, `[download] Downloading playlist: <title>`,
|
||||
# `… Skipping already downloaded playlist: <title>` — the title runs to the end of the line.
|
||||
_TITLE_TAIL_RE = re.compile(r"(playlist: |\[download\] Downloading \w+: ).*", re.I)
|
||||
# An id quoted mid-sentence, not as a prefix: `… not available for <id>`, and `--no-playlist`'s
|
||||
# `Downloading just the video <id>` (`_yes_playlist`) / `Downloading just video <id>` (multifeed).
|
||||
_BARE_ID_RE = re.compile(
|
||||
r"((?:subtitles not available for|Downloading just (?:the )?\w+) )[^\s,;]+"
|
||||
)
|
||||
# Two segments or more, and not glued to a word: `www.youtube.com/…` is what the URL pattern
|
||||
# leaves behind, and it must survive this one.
|
||||
_PATH_RE = re.compile(r"(?<![\w.…])/(?:[^/\s\"']+/)+[^/\s\"']*")
|
||||
|
||||
|
||||
_HIDDEN: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar(
|
||||
"formats_hidden", default=()
|
||||
)
|
||||
|
||||
|
||||
def generic_id(url: str) -> str:
|
||||
"""The name yt-dlp's generic extractor gives a URL: its basename, extension off, percent-
|
||||
decoded — `InfoExtractor._generic_id` (2026.07.04), pinned against it in the tests."""
|
||||
return urllib.parse.unquote(os.path.splitext(url.rstrip("/").split("/")[-1])[0])
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def hiding(url: str | None) -> Iterator[None]:
|
||||
"""For the work on one URL: `log_safe` also takes out the name yt-dlp's generic extractor
|
||||
gives it, exactly, where yt-dlp puts it (`[generic] <name>:`). A context variable, so it
|
||||
holds for yt-dlp's logger calls too — they run on the same thread as the extraction."""
|
||||
name = generic_id(url) if url else ""
|
||||
token = _HIDDEN.set((*_HIDDEN.get(), name) if name else _HIDDEN.get())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_HIDDEN.reset(token)
|
||||
|
||||
|
||||
def log_safe(text: object) -> str:
|
||||
"""yt-dlp's text (its own lines, or a DownloadError's message) with the parts that name a
|
||||
person's download taken out. Every log line that carries yt-dlp's words goes through here,
|
||||
not just the adapter's: yt-dlp raises the same text it has just logged, so a line that skipped
|
||||
it would put back one line later exactly what the adapter took out."""
|
||||
text = str(text)
|
||||
for name in _HIDDEN.get():
|
||||
text = text.replace(f"[generic] {name}:", "[generic] …:")
|
||||
text = _URL_RE.sub(lambda m: f"{m.group(1)}/…", text)
|
||||
text = _ID_PREFIX_RE.sub(r"\1…:", text)
|
||||
text = _GENERIC_ID_RE.sub(r"\1…:", text)
|
||||
text = _PLAYLIST_TITLE_RE.sub(r"\1…", text)
|
||||
text = _TITLE_TAIL_RE.sub(r"\1…", text)
|
||||
text = _BARE_ID_RE.sub(r"\1…", text)
|
||||
return _PATH_RE.sub("…", text)
|
||||
|
||||
|
||||
class _YdlLog:
|
||||
"""yt-dlp's logger protocol, pointed at ours.
|
||||
|
||||
Without it yt-dlp explains itself to nobody: `quiet` sends its output to a stdout we do not
|
||||
read, so "Requested format is not available", a signature-extraction failure or a POT refusal
|
||||
reached us only as whatever text ended up in the raised exception. Measured against yt-dlp
|
||||
2026.07.04: with `logger` set it calls `debug` for extraction and progress lines and
|
||||
`warning`/`error` for the rest, and `quiet`/`no_warnings` do NOT gate those calls.
|
||||
|
||||
`debug` goes to TRACE: it is several lines per second during a download, so it shows only
|
||||
when the whole process runs at TRACE. (Not under a user's own Teljes consent — that lives in
|
||||
the API process's request scope, and the download child that runs yt-dlp has none.)
|
||||
|
||||
`error` goes to WARNING, not ERROR. yt-dlp calls it from `trouble()` just before it raises,
|
||||
on EVERY failed attempt — including the ones the pipeline then recovers from with its fallback
|
||||
client set. Whether the job failed is the pipeline's verdict (`download.job.failed`), and that
|
||||
is the one line whose level should say so."""
|
||||
|
||||
def _emit(self, level: int, msg: str) -> None:
|
||||
# The line goes as an ARGUMENT, not as the format string. yt-dlp writes progress as
|
||||
# `[download] 53.2% of 12.00MiB`, and while stdlib logging only applies `%` formatting to
|
||||
# a record that has args, this way the question never arises — and it is the shape ruff's
|
||||
# `G` rules ask for. (The sink's flood limiter groups a bare `%s` by the text itself, so
|
||||
# distinct yt-dlp warnings do not share one bucket — `logsink._template`.)
|
||||
log.log(level, "%s", log_safe(msg), extra=logs.event("download.ytdlp"))
|
||||
|
||||
def debug(self, msg):
|
||||
self._emit(logs.TRACE, msg)
|
||||
|
||||
def info(self, msg):
|
||||
self._emit(logging.DEBUG, msg)
|
||||
|
||||
def warning(self, msg):
|
||||
self._emit(logging.WARNING, msg)
|
||||
|
||||
def error(self, msg):
|
||||
self._emit(logging.WARNING, msg)
|
||||
|
||||
|
||||
_YDL_LOG = _YdlLog()
|
||||
|
||||
# Fields that affect the produced file — and therefore the cache identity.
|
||||
_SIG_KEYS = (
|
||||
@@ -50,10 +179,16 @@ _DEFAULTS = {
|
||||
# prefer H.264+AAC (browser/iOS-compatible) rather than yt-dlp's default VP9/Opus best.
|
||||
# v3: mp4 compat now *selects* H.264 (avc1) with a progressive fallback + re-encodes the rare
|
||||
# AV1/VP9-only result to H.264, so cached AV1 assets must re-derive (fixes iOS broken-player).
|
||||
_SIG_VERSION = 3
|
||||
# v4: EVERY download is clamped to the mp4/H.264/AAC compat profile (no more mkv/webm/VP9/AV1/Opus
|
||||
# opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped
|
||||
# the v3 net — must re-derive through the current browser-safe guarantee.
|
||||
_SIG_VERSION = 4
|
||||
# Audio-only downloads version on their own, so a change to the audio bytes doesn't re-download every
|
||||
# cached video. v5: an extracted .mp3/.m4a is stored as itself instead of remuxed into an .mp4 that
|
||||
# iPad's WebKit refuses to play, so a cached remuxed audio asset must re-derive.
|
||||
_AUDIO_SIG_VERSION = 5
|
||||
|
||||
_SPONSORBLOCK_CATEGORIES = ["sponsor", "selfpromo", "interaction"]
|
||||
_VCODEC_SORT = {"h264": "vcodec:h264", "vp9": "vcodec:vp9", "av1": "vcodec:av01"}
|
||||
|
||||
|
||||
def normalize(spec: dict) -> dict:
|
||||
@@ -69,46 +204,92 @@ def normalize(spec: dict) -> dict:
|
||||
# rather than raising an unhandled 500 at enqueue.
|
||||
try:
|
||||
s["max_height"] = int(s["max_height"])
|
||||
except (TypeError, ValueError):
|
||||
except (TypeError, ValueError) as exc:
|
||||
# "best" is the opposite of what a height cap asks for: the user set a ceiling to save
|
||||
# bandwidth or disk, and the download quietly takes the largest rendition there is.
|
||||
log.warning(
|
||||
"the height cap on a download could not be read and was dropped",
|
||||
extra=logs.event("download.format.max_height_dropped", reason=type(exc).__name__),
|
||||
)
|
||||
s["max_height"] = None
|
||||
for b in ("embed_subs", "embed_chapters", "embed_thumbnail", "sponsorblock"):
|
||||
s[b] = bool(s[b])
|
||||
# Audio-only files can't embed subtitles/thumbnails as video; keep the flags meaningful.
|
||||
if s["mode"] == "a":
|
||||
s["embed_subs"] = False
|
||||
# Browser-playability guarantee (E7): the Download module exists to download → share → PLAY
|
||||
# in-browser, so every output must play in EVERY browser, iOS/Safari/WebKit included. The only
|
||||
# universally-safe combo is mp4 + H.264 + AAC/MP3, so we CLAMP the spec to it HERE — the single
|
||||
# point every path (picker, stored profile, raw API) flows through — instead of only hiding
|
||||
# options in the UI. Non-mp4 containers (mkv doesn't play at all; webm/VP9/AV1 fail iOS) and any
|
||||
# explicit vcodec are coerced to the compat profile; Opus audio (Safari-flaky as a bare file)
|
||||
# falls back to AAC/m4a. A source that exposes no H.264 is still caught by the worker's
|
||||
# browser-safe re-encode (`_ensure_browser_playable`), so playback is guaranteed end-to-end.
|
||||
s["container"] = "mp4"
|
||||
s["vcodec"] = None
|
||||
if s["audio_format"] not in ("m4a", "mp3"):
|
||||
s["audio_format"] = "m4a"
|
||||
return s
|
||||
|
||||
|
||||
def format_sig(spec: dict, salt: str | None = None) -> str:
|
||||
def _sig_version(s: dict) -> int:
|
||||
return _AUDIO_SIG_VERSION if s["mode"] == "a" else _SIG_VERSION
|
||||
|
||||
|
||||
def format_sig(spec: dict, salt: str | None = None, *, version: int | None = None) -> str:
|
||||
"""Stable short hash of the output-affecting spec — the MediaAsset cache key component.
|
||||
|
||||
`salt` forks a distinct, non-dedup identity for the same spec (used by force-redownload so it
|
||||
always re-fetches into its own asset instead of sharing a cached file with other jobs)."""
|
||||
always re-fetches into its own asset instead of sharing a cached file with other jobs).
|
||||
`version` hashes under a signature version other than the current one (see `format_sigs`)."""
|
||||
s = normalize(spec)
|
||||
data = {k: s[k] for k in _SIG_KEYS}
|
||||
data["_v"] = _SIG_VERSION
|
||||
data["_v"] = _sig_version(s) if version is None else version
|
||||
if salt is not None:
|
||||
data["_salt"] = salt
|
||||
payload = json.dumps(data, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(payload.encode()).hexdigest()[:24]
|
||||
|
||||
|
||||
def format_sigs(spec: dict, salt: str | None = None) -> tuple[str, str]:
|
||||
"""`format_sig` under the current signature version and under the one before it. An asset keeps
|
||||
the signature it was created with, so a job queued before a deploy that bumped the version and
|
||||
run after it downloads into an asset carrying the previous one — which must still be recognised
|
||||
as that user's private asset (`credentials.is_authed_asset`), or it downloads without its cookie."""
|
||||
previous = _sig_version(normalize(spec)) - 1
|
||||
return format_sig(spec, salt), format_sig(spec, salt, version=previous)
|
||||
|
||||
|
||||
def build_ydl_opts(
|
||||
spec: dict,
|
||||
outtmpl: str,
|
||||
progress_hook,
|
||||
extra: dict | None = None,
|
||||
cookiefile: str | None = None,
|
||||
max_bytes: int | None = None,
|
||||
) -> dict:
|
||||
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
|
||||
|
||||
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
|
||||
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
||||
generic options below. Everything here is source-agnostic."""
|
||||
generic options below. Everything here is source-agnostic.
|
||||
|
||||
`cookiefile` is the acting user's per-host cookies.txt path (S6), attached ONLY by the worker
|
||||
after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra`
|
||||
merge so a source adapter can never clobber it.
|
||||
|
||||
`max_bytes` (SEC-13) is the caller's remaining storage budget; yt-dlp refuses a format larger
|
||||
than it rather than filling the disk and only then being caught by the quota accounting."""
|
||||
s = normalize(spec)
|
||||
h = s["max_height"]
|
||||
opts: dict = {
|
||||
"outtmpl": outtmpl,
|
||||
"noplaylist": True,
|
||||
# SEC-13: `noplaylist` only demotes a video-inside-a-playlist URL to the single video. A
|
||||
# URL that is PURELY a playlist or channel has no such video to fall back to, so it still
|
||||
# expands into every entry — one enqueue, an unbounded number of downloads. One job means
|
||||
# one item; a user who wants the whole list enqueues it item by item, under their quota.
|
||||
"playlist_items": "1",
|
||||
"quiet": True,
|
||||
"no_warnings": True,
|
||||
"noprogress": True,
|
||||
@@ -132,6 +313,27 @@ def build_ydl_opts(
|
||||
if extra:
|
||||
opts.update(extra)
|
||||
|
||||
# With a logger attached, `quiet` and `no_warnings` stop deciding whether yt-dlp explains itself
|
||||
# and only decide where — so both stay on above and the explanation comes here instead of to a
|
||||
# stdout nothing reads. AFTER the adapter merge, like `cookiefile` and the size cap: an adapter
|
||||
# must not be able to silence the one account of what went wrong that a failed job leaves.
|
||||
opts["logger"] = _YDL_LOG
|
||||
|
||||
# SEC-13: hard ceiling on a single download. AFTER the adapter merge, for the same reason
|
||||
# `cookiefile` is — a source adapter must not be able to raise a limit the quota layer set.
|
||||
#
|
||||
# A non-positive budget is left unset because 0 is not an expressible yt-dlp filter, NOT because
|
||||
# nothing needs to happen: the worker refuses that job outright before extraction (`_OwnerRefused`
|
||||
# → `quota_exceeded`). Enqueue cannot make that call — it ran while the user was still under the
|
||||
# cap. Delete the worker's refusal and this branch silently uncaps the one user already over it.
|
||||
if max_bytes and max_bytes > 0:
|
||||
opts["max_filesize"] = max_bytes
|
||||
|
||||
# The acting user's per-host cookie (S6). AFTER the adapter merge on purpose: an adapter must not
|
||||
# be able to override the authenticated session the worker resolved for this specific user.
|
||||
if cookiefile:
|
||||
opts["cookiefile"] = cookiefile
|
||||
|
||||
if s["mode"] == "a":
|
||||
opts["format"] = "bestaudio/best"
|
||||
opts["postprocessors"].append(
|
||||
@@ -143,51 +345,33 @@ def build_ydl_opts(
|
||||
)
|
||||
else:
|
||||
hcap = f"[height<=?{h}]" if h else ""
|
||||
# Compatibility-first for mp4 (the universal container): produce H.264 video + AAC audio
|
||||
# so shared/downloaded files play in EVERY browser, including iOS/Safari. WebKit (which
|
||||
# every iOS browser, Brave included, is forced to use) decodes only H.264/H.265 + AAC
|
||||
# inside mp4 — a VP9/AV1-in-mp4 file shows a broken player there, even though Chromium
|
||||
# plays it fine. A custom profile that explicitly sets `vcodec` opts out.
|
||||
compat = s["container"] == "mp4" and s["vcodec"] is None
|
||||
# Every video download is the mp4/H.264 compat profile (normalize() clamps container→mp4,
|
||||
# vcodec→None), so shared/downloaded files play in EVERY browser, iOS/Safari included. WebKit
|
||||
# (which every iOS browser, Brave included, is forced to use) decodes only H.264/H.265 + AAC
|
||||
# inside mp4 — a VP9/AV1-in-mp4 file shows a broken player there — so H.264 is selected at the
|
||||
# SELECTION level, not merely preferred: a sort key can't conjure an H.264 stream a source
|
||||
# doesn't expose as video-only. A source that ships only non-H.264 is caught after download by
|
||||
# the worker's browser-safe re-encode (`_ensure_browser_playable`).
|
||||
if s["mode"] == "v":
|
||||
if compat:
|
||||
# Video-only: keep it audio-free. Prefer an H.264 *video-only* stream, then any
|
||||
# video-only, and only fall to a progressive (audio-bearing) `best` as a last
|
||||
# resort — same audio-free-when-possible guarantee as the old `bestvideo/best`.
|
||||
opts["format"] = (
|
||||
f"bestvideo[vcodec^=avc1]{hcap}/bestvideo{hcap}"
|
||||
f"/best[vcodec^=avc1]{hcap}/best{hcap}"
|
||||
)
|
||||
else:
|
||||
opts["format"] = f"bestvideo{hcap}/best{hcap}"
|
||||
# Video-only: keep it audio-free. Prefer an H.264 video-only stream, then any video-only,
|
||||
# and only fall to a progressive (audio-bearing) `best` as a last resort.
|
||||
opts["format"] = (
|
||||
f"bestvideo[vcodec^=avc1]{hcap}/bestvideo{hcap}/best[vcodec^=avc1]{hcap}/best{hcap}"
|
||||
)
|
||||
else: # av
|
||||
if compat:
|
||||
# Prefer H.264 at the SELECTION level, not just via format_sort: a sort key is only
|
||||
# a *preference* and can't conjure an H.264 stream a source doesn't expose as
|
||||
# video-only. A non-YouTube source (e.g. Facebook) may ship AV1 as the best
|
||||
# video-only stream and H.264 only as a *progressive* (a+v) format — which
|
||||
# `bestvideo` skips — so `bestvideo+bestaudio` picked AV1 and iOS broke. This chain
|
||||
# tries avc1 video-only, then the avc1 progressive, then anything.
|
||||
opts["format"] = (
|
||||
f"bestvideo[vcodec^=avc1]{hcap}+bestaudio[acodec^=mp4a]"
|
||||
f"/best[vcodec^=avc1]{hcap}"
|
||||
f"/bestvideo{hcap}+bestaudio/best{hcap}"
|
||||
)
|
||||
else:
|
||||
opts["format"] = f"bestvideo{hcap}+bestaudio/best{hcap}"
|
||||
if s["container"]:
|
||||
opts["merge_output_format"] = s["container"]
|
||||
sort = []
|
||||
# Keep the codec preference as a tiebreaker (ranks H.264/AAC above resolution among the
|
||||
# selected candidates); the selector above is the hard guarantee, this is belt-and-braces.
|
||||
if compat:
|
||||
sort += ["vcodec:h264", "acodec:aac"]
|
||||
# avc1 video-only + m4a audio, then the avc1 progressive, then anything (re-encoded).
|
||||
opts["format"] = (
|
||||
f"bestvideo[vcodec^=avc1]{hcap}+bestaudio[acodec^=mp4a]"
|
||||
f"/best[vcodec^=avc1]{hcap}"
|
||||
f"/bestvideo{hcap}+bestaudio/best{hcap}"
|
||||
)
|
||||
opts["merge_output_format"] = "mp4"
|
||||
# Codec preference tiebreaker (ranks H.264/AAC among the selected candidates); the selector
|
||||
# above is the hard guarantee, this is belt-and-braces.
|
||||
sort = ["vcodec:h264", "acodec:aac"]
|
||||
if h:
|
||||
sort.append(f"res:{h}")
|
||||
if s["vcodec"] in _VCODEC_SORT:
|
||||
sort.append(_VCODEC_SORT[s["vcodec"]])
|
||||
if sort:
|
||||
opts["format_sort"] = sort
|
||||
opts["format_sort"] = sort
|
||||
|
||||
# SponsorBlock first (marks segments), then strip them from the media + chapters.
|
||||
if s["sponsorblock"]:
|
||||
|
||||
@@ -16,7 +16,6 @@ Five passes each run:
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
@@ -24,14 +23,13 @@ from pathlib import Path
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logging.getLogger("siftlode.downloads.gc")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
||||
@@ -86,18 +84,35 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
|
||||
cutoff = time.time() - _ORPHAN_MIN_AGE_S
|
||||
removed = 0
|
||||
# Throttle the whole-tree walk: this runs in the API process, and an unpaced rglob+stat over a
|
||||
# large download root monopolizes disk IO long enough to make web requests + playback stutter.
|
||||
# Yield the IO channel after every `batch` files touched so foreground work interleaves.
|
||||
pause_s = max(0, settings.download_gc_sweep_pause_ms) / 1000.0
|
||||
batch = max(1, settings.download_gc_sweep_batch)
|
||||
touched = 0
|
||||
|
||||
def _throttle() -> None:
|
||||
nonlocal touched
|
||||
touched += 1
|
||||
if pause_s and touched % batch == 0:
|
||||
time.sleep(pause_s)
|
||||
|
||||
unreclaimed = 0
|
||||
for entry in root.iterdir():
|
||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
||||
continue
|
||||
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
||||
for f in files:
|
||||
_throttle()
|
||||
try:
|
||||
if f.resolve() in keep or f.stat().st_mtime > cutoff:
|
||||
continue
|
||||
f.unlink()
|
||||
removed += 1
|
||||
except OSError:
|
||||
pass
|
||||
# silent-ok: counted, not logged per file — a broken mount would put one line
|
||||
# under every orphan. The total is reported once at the end of the sweep.
|
||||
unreclaimed += 1
|
||||
if entry.is_dir(): # prune emptied dirs (deepest first), then the channel dir itself
|
||||
for d in sorted(
|
||||
(p for p in entry.rglob("*") if p.is_dir()),
|
||||
@@ -106,12 +121,20 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
):
|
||||
try:
|
||||
d.rmdir()
|
||||
# silent-ok: `rmdir` refusing means the directory still holds files, which is
|
||||
# the ordinary answer here — this prune only takes the ones that emptied.
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
entry.rmdir()
|
||||
# silent-ok: the same prune one level up, at the channel directory.
|
||||
except OSError:
|
||||
pass
|
||||
if unreclaimed:
|
||||
log.warning(
|
||||
"some orphaned download files could not be reclaimed",
|
||||
extra=logs.event("download.gc.unreclaimed", files=unreclaimed, removed=removed),
|
||||
)
|
||||
return removed
|
||||
|
||||
|
||||
|
||||
@@ -49,6 +49,8 @@ def check_grant(token: str, grant: str | None, password_version: int) -> bool:
|
||||
exp_s, sig = grant.split(".", 1)
|
||||
try:
|
||||
exp = int(exp_s)
|
||||
# silent-ok: an untrusted grant that does not parse is simply not a valid grant, and the
|
||||
# honest answer to it — a password prompt — is the same as for one that expired.
|
||||
except ValueError:
|
||||
return False
|
||||
if exp < datetime.now(timezone.utc).timestamp():
|
||||
@@ -80,9 +82,13 @@ def public_meta(
|
||||
|
||||
Resolves the owner's per-download display overrides (custom title/channel) over the asset's
|
||||
auto-extracted values, and surfaces the source + any extra reference links as clickable URLs."""
|
||||
from app.downloads import service
|
||||
from app.downloads import mediakind, service
|
||||
|
||||
source_url = service.reference_url(job, asset)
|
||||
# Audio-only assets (mp3 rips, extracted audio) can't play inside a <video> element on WebKit
|
||||
# (all iOS browsers), so the watch page renders a native <audio> instead. A generic uploaded
|
||||
# file (media_kind='file') has no player at all — the watch page shows file info + a download.
|
||||
# Both branches read the stored media_kind (the single source), not a re-derived codec check.
|
||||
return {
|
||||
"title": (job and job.display_name) or asset.title,
|
||||
"uploader": (job and job.display_uploader) or asset.uploader,
|
||||
@@ -92,6 +98,10 @@ def public_meta(
|
||||
"duration_s": asset.duration_s,
|
||||
"width": asset.width,
|
||||
"height": asset.height,
|
||||
"audio": asset.media_kind == mediakind.AUDIO,
|
||||
"media_kind": asset.media_kind,
|
||||
"size_bytes": asset.size_bytes,
|
||||
"container": asset.container,
|
||||
"allow_download": link.allow_download,
|
||||
"file_url": file_url,
|
||||
"poster_url": poster_url,
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""The media-kind discriminator for a library asset — the single home of one invariant.
|
||||
|
||||
`MediaAsset.media_kind` is `"video" | "audio" | "file"`. It decides which library actions apply:
|
||||
media (video/audio) gets play/edit/share; a generic uploaded file gets download + share only, and
|
||||
is never rendered inline (see the force-download serving in the file/watch routes). Every finalize
|
||||
path computes it HERE — the download worker + the upload ingest via `derive`, the editor by
|
||||
carrying the source's kind over — so the rule lives in code, not in a comment that can drift (an
|
||||
earlier ad-hoc `vcodec is None and acodec is not None` audio check lived only in links.py).
|
||||
"""
|
||||
|
||||
VIDEO = "video"
|
||||
AUDIO = "audio"
|
||||
FILE = "file"
|
||||
|
||||
|
||||
def derive(vcodec: str | None, acodec: str | None) -> str:
|
||||
"""Media kind from the probed codecs: a video stream -> video, audio-only -> audio, neither
|
||||
(a non-media upload: pdf/zip/docx/…) -> file."""
|
||||
if vcodec:
|
||||
return VIDEO
|
||||
if acodec:
|
||||
return AUDIO
|
||||
return FILE
|
||||
@@ -66,6 +66,8 @@ def _jpeg_size(path: Path) -> tuple[int, int] | None:
|
||||
width = (data[3] << 8) + data[4]
|
||||
return width, height
|
||||
f.seek(length - 2, 1) # skip this segment's payload
|
||||
# silent-ok: None omits width/height from the unfurl card, which is what the card does for
|
||||
# every poster whose dimensions were never asked for. Nothing is claimed that is not true.
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Child-side download/edit pipeline — the yt-dlp download + ffmpeg edit that runs in the spawned subprocess."""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
@@ -9,12 +8,13 @@ from pathlib import Path
|
||||
|
||||
import yt_dlp
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import adapters, formats, service, storage
|
||||
from app.downloads import adapters, credentials, formats, mediakind, probe, quota, service, storage
|
||||
from app.downloads import urlguard
|
||||
from app.downloads.jobdb import (
|
||||
_edit_staging_dir,
|
||||
_job_status,
|
||||
@@ -23,18 +23,31 @@ from app.downloads.jobdb import (
|
||||
_staging_dir,
|
||||
_touch_job,
|
||||
)
|
||||
from app.downloads.stated import StatedFailure
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
|
||||
log = logging.getLogger("siftlode.worker")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class _Aborted(Exception):
|
||||
"""Raised inside the progress hook when the job was paused/canceled mid-download."""
|
||||
|
||||
|
||||
class _OwnerRefused(Exception):
|
||||
"""A refusal that belongs to THIS job's owner, not to the source.
|
||||
|
||||
MediaAssets are shared: two users asking for the same video and format get one row (see
|
||||
`service.get_or_create_asset`), and the generic failure handler writes the message onto it —
|
||||
correctly, because an upstream failure really is everyone's. A per-user refusal is not. Written
|
||||
to the shared row it would fail a second user's queued job for the same video with the first
|
||||
user's private reason (`worker._process_job` reads `asset.error` verbatim), telling someone with
|
||||
an empty library to free up space. So this is caught separately: the job fails, the asset goes
|
||||
back to `pending` for whoever else is holding it."""
|
||||
|
||||
|
||||
# Both progress sources call back FAR faster than the database should be touched: yt-dlp fires its
|
||||
# hook per chunk, and ffmpeg's `-progress pipe:1` emits a block of ~12 lines every ~0.5s while
|
||||
# run_ffmpeg calls its callbacks once per LINE. Every callback here is a round-trip (`_job_status`
|
||||
@@ -85,6 +98,8 @@ def _parse_upload_date(raw) -> date | None:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(str(raw), "%Y%m%d").date()
|
||||
# silent-ok: a date we cannot read becomes a date the card does not show, which is what the
|
||||
# card shows for every source that never supplies one. Absent is the honest rendering.
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@@ -100,9 +115,10 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
# `or info.get("id")` avoids a blank card while downloading when the cleaned title is empty
|
||||
# (e.g. a title that was only a social engagement prefix); the completion path later sets
|
||||
# the final title with its own source_ref fallback.
|
||||
asset.title = normalize_title(info.get("title")) or info.get("id")
|
||||
adapter = adapters.resolve_adapter(None, None, info)
|
||||
asset.title = normalize_title(adapter.title(info)) or info.get("id")
|
||||
asset.uploader = info.get("uploader") or info.get("channel")
|
||||
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
|
||||
asset.uploader_url = adapter.uploader_url(info)
|
||||
asset.thumbnail_url = info.get("thumbnail")
|
||||
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
||||
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
||||
@@ -124,7 +140,14 @@ def _make_progress_hook(job_id: int, asset_id: int):
|
||||
try:
|
||||
_fill_asset_meta_early(asset_id, d["info_dict"])
|
||||
except Exception: # noqa: BLE001 — metadata is best-effort, never fail a download
|
||||
pass
|
||||
# Best-effort is about the DOWNLOAD, not about knowing. What this swallows is
|
||||
# anything from a shape change in yt-dlp's info dict to the database being down,
|
||||
# and all the user sees is a card with an id on it until the job finishes.
|
||||
log.info(
|
||||
"early metadata could not be filled in for this download",
|
||||
extra=logs.event("download.meta.early_fill_failed", asset_id=asset_id),
|
||||
exc_info=True,
|
||||
)
|
||||
if d.get("status") != "downloading":
|
||||
return
|
||||
if not allow_write():
|
||||
@@ -285,7 +308,7 @@ def _produced_file(info: dict, staging: Path) -> Path:
|
||||
and not _YTDLP_TEMP_RE.search(p.name)
|
||||
]
|
||||
if not cands:
|
||||
raise RuntimeError("yt-dlp produced no output file")
|
||||
raise StatedFailure("yt-dlp produced no output file")
|
||||
return max(cands, key=lambda p: p.stat().st_size)
|
||||
|
||||
|
||||
@@ -304,12 +327,38 @@ def _media_info(info: dict) -> dict:
|
||||
return {**info, **chosen}
|
||||
|
||||
|
||||
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
def _extract_with_fallback(
|
||||
job_id, asset_id, url, spec, staging, adapter, cookiefile=None, max_bytes=None
|
||||
):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
|
||||
client_sets = adapter.client_sets()
|
||||
error); a Facebook download WITH a cookie is retried as Chrome behind "Cannot parse data"; other
|
||||
sources make a single attempt. Returns the info dict of the successful attempt.
|
||||
|
||||
Before the first attempt the adapter may resolve `url` to the link that really holds the video (a
|
||||
Facebook group post → its video's watch link); when the page shows no single video there, or is not
|
||||
the post at all (a refused cookie's login page), the adapter raises its own verdict
|
||||
(adapters.POST_*_MESSAGE) instead of letting yt-dlp guess, and a failed page read fails the
|
||||
download as itself.
|
||||
|
||||
`cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated.
|
||||
|
||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt."""
|
||||
client_sets = adapter.client_sets(authed=cookiefile is not None)
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
# Resolving may read a page first — network time the stall watchdog must not see as a stall.
|
||||
_touch_job(job_id)
|
||||
target = adapter.resolve_url(
|
||||
url,
|
||||
formats.build_ydl_opts(
|
||||
spec,
|
||||
outtmpl,
|
||||
_make_progress_hook(job_id, asset_id),
|
||||
cookiefile=cookiefile,
|
||||
max_bytes=max_bytes,
|
||||
),
|
||||
)
|
||||
last_exc: Exception | None = None
|
||||
for i, clients in enumerate(client_sets):
|
||||
opts = formats.build_ydl_opts(
|
||||
@@ -317,6 +366,8 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
outtmpl,
|
||||
_make_progress_hook(job_id, asset_id),
|
||||
extra=adapter.ydl_opts_extra(clients),
|
||||
cookiefile=cookiefile,
|
||||
max_bytes=max_bytes,
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||
@@ -325,7 +376,7 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
# killed — acceptable, as socket_timeout + extractor_retries bound one attempt well under it.)
|
||||
try:
|
||||
with yt_dlp.YoutubeDL(opts) as ydl:
|
||||
return ydl.extract_info(url, download=True)
|
||||
return ydl.extract_info(target, download=True)
|
||||
except yt_dlp.utils.DownloadError as exc:
|
||||
last_exc = exc
|
||||
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||
@@ -333,13 +384,22 @@ def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
"job %s: clients %s failed (%s); retrying with fallback",
|
||||
job_id,
|
||||
clients,
|
||||
str(exc)[:70],
|
||||
formats.log_safe(exc)[:70],
|
||||
)
|
||||
for p in list(staging.iterdir()): # drop partials before the retry
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
except OSError as exc:
|
||||
# A partial left behind is not inert: the next client set resumes into it,
|
||||
# so the retry this loop exists to make clean starts dirty instead.
|
||||
log.info(
|
||||
"a partial file could not be removed before the retry",
|
||||
extra=logs.event(
|
||||
"download.retry.partial_unlink_failed",
|
||||
job_id=job_id,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
_set_job(job_id, progress=0, phase="downloading")
|
||||
continue
|
||||
raise
|
||||
@@ -362,8 +422,12 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
or just remux when the codecs are fine but the container isn't. **Scope:** only the mp4 compat
|
||||
profile is normalized; an explicit `vcodec` or a non-mp4 custom profile is the user opting out
|
||||
of the guarantee and is left as-is (so a `webm` profile can still ship VP9/Opus). Returns the
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk."""
|
||||
if spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk.
|
||||
|
||||
An audio download (mode "a") is never touched: yt-dlp already extracted it to .mp3 / .m4a, which
|
||||
every browser plays as audio/mpeg / audio/mp4. Remuxing it into .mp4 served an mp3 as video/mp4,
|
||||
which iPad's WebKit refuses to play (prod job 268)."""
|
||||
if spec.get("mode") == "a" or spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
return produced
|
||||
vcodec, acodec = editmod.probe_codecs(produced)
|
||||
is_mp4 = produced.suffix.lower() == ".mp4"
|
||||
@@ -381,7 +445,9 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
# re-encode still shows real progress instead of a frozen 0%.
|
||||
total_s = int(info["duration"]) if info.get("duration") else editmod.probe_duration(produced)
|
||||
editmod.run_ffmpeg(
|
||||
editmod.build_browser_safe_cmd(produced, dest, reenc_v, reenc_a),
|
||||
editmod.build_browser_safe_cmd(
|
||||
produced, dest, reenc_v, reenc_a, threads=editmod.encode_thread_cap()
|
||||
),
|
||||
total_s=total_s,
|
||||
on_progress=_ffmpeg_progress_writer(job_id, "processing"),
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
@@ -395,8 +461,12 @@ def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict
|
||||
return dest
|
||||
|
||||
|
||||
def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict) -> None:
|
||||
def _download(
|
||||
job_id: int, asset_id: int, source_kind: str, source_ref: str, spec: dict, user_id: int
|
||||
) -> None:
|
||||
staging = _staging_dir(asset_id)
|
||||
cookiefile: Path | None = None
|
||||
url: str | None = None # read by the failure branch, which can run before it is resolved
|
||||
try:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
@@ -405,7 +475,48 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
_set_job(job_id, phase="downloading", progress=0)
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
||||
url = service.source_url(source_kind, source_ref)
|
||||
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
|
||||
# SEC-12, second half: the enqueue guard ran when the job was created, which may have been
|
||||
# hours ago and against a different DNS answer. This is where the socket actually gets
|
||||
# opened, so the check is repeated here and the job fails rather than fetching.
|
||||
if source_kind == "url":
|
||||
try:
|
||||
urlguard.check_url(url)
|
||||
except urlguard.BlockedUrl as e:
|
||||
raise StatedFailure(f"Refusing to download from this address ({e.reason}).") from e
|
||||
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
remaining_bytes: int | None = None
|
||||
with SessionLocal() as cdb:
|
||||
target = cdb.get(MediaAsset, asset_id)
|
||||
if target is not None:
|
||||
cookiefile = credentials.resolve_cookiefile(
|
||||
cdb, user_id, url, target, spec, staging
|
||||
)
|
||||
# SEC-13: what this user has left. `check_enqueue` only compared the footprint as it
|
||||
# stood when the job was QUEUED — a download's size is unknown until it finishes, so a
|
||||
# single job could previously blow through the cap by any amount. Handing the remainder
|
||||
# to yt-dlp as `max_filesize` makes the format selector refuse an oversized file up
|
||||
# front instead.
|
||||
remaining_bytes = quota.remaining_bytes(cdb, user_id)
|
||||
# …and zero is a REFUSAL, not "no limit". Earlier jobs of this user may have filled the cap
|
||||
# while this one sat in the queue, and 0 is not an expressible yt-dlp `max_filesize`, so
|
||||
# passing it down would leave the ceiling unset — the one user already at their cap would be
|
||||
# the only one downloading unbounded. Fail the job here instead. Not retryable until they
|
||||
# free space, which is what the `quota_exceeded` category tells them.
|
||||
if remaining_bytes == 0:
|
||||
raise _OwnerRefused(dlerrors.QUOTA_FULL_MESSAGE)
|
||||
info = _media_info(
|
||||
_extract_with_fallback(
|
||||
job_id,
|
||||
asset_id,
|
||||
url,
|
||||
spec,
|
||||
staging,
|
||||
adapter,
|
||||
cookiefile=str(cookiefile) if cookiefile else None,
|
||||
max_bytes=remaining_bytes,
|
||||
)
|
||||
)
|
||||
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
||||
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||
@@ -413,13 +524,19 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
produced = _produced_file(info, staging)
|
||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||
ext = produced.suffix.lstrip(".").lower()
|
||||
# Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's
|
||||
# `info` dict, whose fields are absent for some sources (Facebook among them): deriving
|
||||
# media_kind from those nulls misclassified a real H.264 video as a download-only 'file', and a
|
||||
# missing duration hid the trim action. Probed before the sidecars so the .nfo runtime has it
|
||||
# too. Same probe the upload ingest uses.
|
||||
vcodec, acodec, width, height, duration_s = probe.resolve_media_facts(produced, info)
|
||||
meta = storage.MediaMeta(
|
||||
video_id=info.get("id") or source_ref,
|
||||
title=normalize_title(info.get("title")) or source_ref,
|
||||
title=normalize_title(adapter.title(info)) or source_ref,
|
||||
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
||||
upload_date=_parse_upload_date(info.get("upload_date")),
|
||||
uniqueid_type=adapter.nfo_uniqueid_type(),
|
||||
duration_s=int(info["duration"]) if info.get("duration") else None,
|
||||
duration_s=duration_s,
|
||||
description=info.get("description"),
|
||||
thumbnail_url=info.get("thumbnail"),
|
||||
)
|
||||
@@ -445,15 +562,14 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = size
|
||||
asset.duration_s = meta.duration_s
|
||||
asset.width = int(info["width"]) if info.get("width") else None
|
||||
asset.height = int(info["height"]) if info.get("height") else None
|
||||
asset.vcodec, asset.acodec, asset.width, asset.height = (
|
||||
vcodec,
|
||||
acodec,
|
||||
width,
|
||||
height,
|
||||
)
|
||||
asset.container = ext
|
||||
asset.vcodec = (
|
||||
info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||
)
|
||||
asset.acodec = (
|
||||
info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
)
|
||||
asset.media_kind = mediakind.derive(asset.vcodec, asset.acodec)
|
||||
asset.title = meta.title
|
||||
asset.uploader = meta.uploader
|
||||
asset.uploader_url = adapter.uploader_url(info)
|
||||
@@ -488,37 +604,130 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, meta.title, job_id, kind="download")
|
||||
log.info("job %s done: %s", job_id, rel)
|
||||
# The asset id, not `rel`: the path is built from the channel and the title.
|
||||
log.info(
|
||||
"job %s done: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event("download.job.done", job_id=job_id, asset_id=asset_id),
|
||||
)
|
||||
|
||||
except (_Aborted, editmod.EditAborted):
|
||||
# Paused/canceled mid-download or mid-transcode: free the asset for a later retry; keep the
|
||||
# job's status (the route set it to paused/canceled). ref_count is adjusted by the route.
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info("job %s aborted (pause/cancel)", job_id)
|
||||
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
|
||||
log.info(
|
||||
"job %s aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("download.job.aborted", job_id=job_id),
|
||||
)
|
||||
except _OwnerRefused as exc:
|
||||
# Fail this job only. The asset is never marked `error`; it goes back to `pending` (below),
|
||||
# so another user's job for the same video is claimed and downloaded normally instead of
|
||||
# inheriting a reason that is about someone else's account.
|
||||
msg = str(exc)[:500]
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
|
||||
# with `error` (error also holds the asset → would re-open the double-release class).
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
error_detail=None,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("job %s failed: %s", job_id, msg)
|
||||
# Released AFTER the job is terminal: the moment the asset is `pending` another job can claim
|
||||
# it, and this one must already be out of the way. (The staging tree is still torn down in
|
||||
# `finally` below — the same narrow window the pause/cancel path above has always had.)
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info(
|
||||
"job %s refused for its owner: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra=logs.event("download.job.refused", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface any failure to the user, keep worker alive
|
||||
msg = str(exc)[:500]
|
||||
# The catalog needs the link and whether the user's cookie went with it: the same yt-dlp
|
||||
# text is Facebook's login wall without one and a refused cookie with one.
|
||||
diagnosis = dlerrors.diagnose(msg, url=url, cookie_attached=cookiefile is not None)
|
||||
# SEC-12: redact a transport-level readout for user-supplied URLs (see user_error_text).
|
||||
msg = dlerrors.user_error_text(source_kind, msg, diagnosis.code)
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never
|
||||
# overwritten with `error` (error also holds the asset → would re-open the
|
||||
# double-release class).
|
||||
service.finalize_if_running(
|
||||
db,
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=diagnosis.code,
|
||||
error_detail=diagnosis.detail,
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
# After the write: a job the write could not fail is not a failed job. When it raises,
|
||||
# the runner reports this failure with it (`runner._run`) and the worker fails the job.
|
||||
# silent-ok: not silent — `_log_download_failed` is the report.
|
||||
_log_download_failed(job_id, msg, diagnosis.code, exc)
|
||||
finally:
|
||||
# Unlink the decrypted cookie file explicitly (the rmtree below removes the whole staging
|
||||
# tree too, but never leave a plaintext session cookie on disk a moment longer than needed).
|
||||
if cookiefile is not None:
|
||||
cookiefile.unlink(missing_ok=True)
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
def _failure_stack(exc: BaseException) -> dict[str, str]:
|
||||
"""The `extra=` fragment that puts a failure's traceback on its line — or nothing.
|
||||
|
||||
Nothing for yt-dlp's DownloadError and for our own `StatedFailure` (`ffmpeg failed (1): …`):
|
||||
their message is the whole story, and their stack is only the `except` that caught them.
|
||||
Everything else keeps it — a TypeError on an odd info dict, a plain RuntimeError, an OSError,
|
||||
a database error: that is our code or our platform failing, and the stack is the lead.
|
||||
|
||||
Scrubbed, link by link (`logs.scrubbed_stack`): a stack's messages are data — an OSError names
|
||||
its file, and a database error raised inside `except DownloadError` carries yt-dlp's
|
||||
unscrubbed text as its context."""
|
||||
if isinstance(exc, (yt_dlp.utils.DownloadError, StatedFailure)):
|
||||
return {}
|
||||
return logs.scrubbed_stack(exc, formats.log_safe)
|
||||
|
||||
|
||||
def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseException) -> None:
|
||||
"""The child's `download.job.failed` line. `msg` is the user's copy and keeps their URL; the
|
||||
log's copy goes through `log_safe`, or it would put back one line later the URL and id the
|
||||
yt-dlp adapter took out of the line before it. The stack only when the classifier could not
|
||||
place the failure: a recognised one is fully described by its code."""
|
||||
unplaced = code in (None, "", "unknown")
|
||||
log.warning(
|
||||
"job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra={
|
||||
**logs.event(
|
||||
"download.job.failed",
|
||||
job_id=job_id,
|
||||
error_code=code,
|
||||
error_type=type(exc).__name__,
|
||||
),
|
||||
**(_failure_stack(exc) if unplaced else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None = None) -> None:
|
||||
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
|
||||
where its line and its event code live. `exc` is the exception when there was one.
|
||||
|
||||
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
|
||||
Logged after the write, as on the download error path."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
@@ -534,7 +743,22 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("edit job %s failed: %s", job_id, msg)
|
||||
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
||||
# built from the channel and the title.
|
||||
log.warning(
|
||||
"edit job %s failed: %s",
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra={
|
||||
**logs.event(
|
||||
"edit.job.failed",
|
||||
job_id=job_id,
|
||||
asset_id=asset_id,
|
||||
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
||||
),
|
||||
**(_failure_stack(exc) if exc is not None else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
@@ -586,7 +810,7 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
)
|
||||
if not dest_staging.exists():
|
||||
raise RuntimeError("ffmpeg produced no output file")
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
|
||||
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||
storage.place_file(dest_staging, settings.download_root, rel)
|
||||
@@ -622,13 +846,24 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
db.commit()
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, dl_title, job_id, kind="edit")
|
||||
log.info("edit job %s done: %s", job_id, rel)
|
||||
# The asset id, not `rel`: the path is built from the channel and the title.
|
||||
log.info(
|
||||
"edit job %s done: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event("edit.job.done", job_id=job_id, asset_id=asset_id),
|
||||
)
|
||||
|
||||
except editmod.EditAborted:
|
||||
# Paused/canceled mid-encode: free the asset for a later retry (ref_count is the route's job).
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info("edit job %s aborted (pause/cancel)", job_id)
|
||||
log.info(
|
||||
"edit job %s aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("edit.job.aborted", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500])
|
||||
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Ground-truth media probe for a produced library file — the SINGLE place both ingest paths ask
|
||||
"what is actually on disk?".
|
||||
|
||||
`media_kind` (and the stored `vcodec`/`acodec`/`width`/`height`) must reflect the real file, not an
|
||||
extractor's metadata: yt-dlp's `info` dict omits codec fields for some sources (e.g. Facebook), and
|
||||
deriving the kind from those nulls misclassified a genuine H.264 video as a download-only `file`
|
||||
(no in-app play). So the download worker (`pipeline`) and the local upload ingest (`upload`) both
|
||||
probe the finished file here; `mediakind.derive` then turns the probed codecs into the kind.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from app.downloads import edit as editmod
|
||||
|
||||
# ffprobe container formats that mean "this is a still image / image sequence", NOT playable media.
|
||||
# A standalone jpg/png/gif/webp/bmp reports a real video stream (codec mjpeg/png/…) with no
|
||||
# `attached_pic` flag, so without this guard it would classify as media_kind='video' and get a Play
|
||||
# button + a broken in-app <video> player (and inline serving on the public share). Keyed on the
|
||||
# container format so an animated gif/webp also lands as a generic 'file' (no browser plays those in
|
||||
# a <video> either). SVG and anything else ffprobe can't decode already fails the probe → 'file'.
|
||||
_IMAGE_FORMATS = {
|
||||
"image2",
|
||||
"image2pipe",
|
||||
"png_pipe",
|
||||
"mjpeg",
|
||||
"mjpeg_pipe",
|
||||
"jpeg_pipe",
|
||||
"gif",
|
||||
"webp_pipe",
|
||||
"bmp_pipe",
|
||||
"tiff_pipe",
|
||||
"ico",
|
||||
"svg_pipe",
|
||||
}
|
||||
|
||||
|
||||
def _facts_from_probe(
|
||||
data: dict,
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) parsed from a NON-None ffprobe JSON dict. A still
|
||||
image / image-sequence container yields no vcodec (→ generic 'file'); a video stream flagged
|
||||
`attached_pic` (embedded cover art in an mp3/m4a) is skipped — it's a thumbnail, not a real video
|
||||
track, so an audio file with album art still classifies as audio."""
|
||||
fmt = data.get("format") or {}
|
||||
fmt_names = {n.strip() for n in (fmt.get("format_name") or "").split(",")}
|
||||
if fmt_names & _IMAGE_FORMATS:
|
||||
return None, None, None, None, None
|
||||
|
||||
vcodec = acodec = None
|
||||
width = height = None
|
||||
for stream in data.get("streams", []):
|
||||
kind = stream.get("codec_type")
|
||||
if kind == "video" and vcodec is None:
|
||||
if (stream.get("disposition") or {}).get("attached_pic"):
|
||||
continue # cover art, not a video track
|
||||
vcodec = stream.get("codec_name")
|
||||
width = stream.get("width")
|
||||
height = stream.get("height")
|
||||
elif kind == "audio" and acodec is None:
|
||||
acodec = stream.get("codec_name")
|
||||
|
||||
duration = None
|
||||
try:
|
||||
raw = fmt.get("duration")
|
||||
if raw is not None:
|
||||
duration = int(float(raw))
|
||||
# silent-ok: an unusable duration is stored as no duration, which the UI renders as unknown
|
||||
# rather than as zero — the honest state, and the one a re-probe can still fill in.
|
||||
except (TypeError, ValueError):
|
||||
duration = None
|
||||
return vcodec, acodec, width, height, duration
|
||||
|
||||
|
||||
def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
|
||||
or the file isn't playable media (an unprobeable file — or a still image — is treated as a generic
|
||||
non-media file, so it becomes a download-only 'file' rather than a broken 'video')."""
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
if data is None:
|
||||
return None, None, None, None, None
|
||||
return _facts_from_probe(data)
|
||||
|
||||
|
||||
def resolve_media_facts(
|
||||
path: Path, info: dict
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) for the stored download asset: PROBE the finished
|
||||
file on disk (the ground truth). yt-dlp's `info` omits codec/dimension fields for some sources
|
||||
(Facebook among them), so deriving media_kind from it there labelled a real H.264 video a
|
||||
download-only 'file'. It omits the duration too (a Facebook reel, an Instagram reel), and a NULL
|
||||
duration hides the Library row's trim action although the file plays for minutes.
|
||||
|
||||
Precedence is by whether the probe RAN, not per-field: if ffprobe read the file, its codec verdict
|
||||
is authoritative — INCLUDING a deliberate None (cover-art-only audio, a still image). A per-field
|
||||
`probe or info` fallback would let `info`'s stale vcodec resurrect a video track the file doesn't
|
||||
have (yt-dlp reports a progressive `vcodec` for an audio-mode download whose video it then strips),
|
||||
mislabelling an audio download as a broken in-app 'video'. Only when ffprobe could NOT read the
|
||||
file at all do we fall back to `info` (the pre-probe behaviour, a safety net); the cosmetic
|
||||
dimensions and the duration still gap-fill from `info` in either case."""
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
info_v = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||
info_a = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
info_w = int(info["width"]) if info.get("width") else None
|
||||
info_h = int(info["height"]) if info.get("height") else None
|
||||
info_d = int(info["duration"]) if info.get("duration") else None
|
||||
if data is None:
|
||||
# ffprobe could not read the file — fall back to the extractor's info entirely.
|
||||
return info_v, info_a, info_w, info_h, info_d
|
||||
pv, pa, pw, ph, pd = _facts_from_probe(data)
|
||||
# Probe ran → its codecs win outright (a None means "no such track", not "unknown"); info only
|
||||
# gap-fills the width/height/duration the probe couldn't read.
|
||||
return (
|
||||
pv,
|
||||
pa,
|
||||
pw if pw is not None else info_w,
|
||||
ph if ph is not None else info_h,
|
||||
pd if pd is not None else info_d,
|
||||
)
|
||||
@@ -7,6 +7,13 @@ Limits come from a per-user DownloadQuota row if the admin set one, else the sys
|
||||
|
||||
Enforcement split: max_jobs + max_bytes are checked at enqueue (they bound holdings); per-user
|
||||
max_concurrent is enforced by the worker at claim time (it bounds simultaneous downloads).
|
||||
|
||||
DOWNLOAD jobs additionally consult `remaining_bytes` in the worker, because enqueue can only compare
|
||||
the footprint as it stood when the job was created and earlier jobs may have filled the cap since.
|
||||
That is a per-job ceiling, not an aggregate ledger: concurrent jobs of one user each read the same
|
||||
remainder (in-flight bytes are not `ready`, so `footprint` cannot see them), so a wave of
|
||||
max_concurrent downloads can still land over the cap — the deliberate trade, since a download's size
|
||||
is unknown until it finishes. Edit jobs do not consult it at all (tracked as SEC-19).
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
@@ -122,6 +129,37 @@ def check_enqueue(db: Session, user_id: int) -> None:
|
||||
raise QuotaExceeded("max_bytes", lim.max_bytes, used)
|
||||
|
||||
|
||||
def remaining_bytes(db: Session, user_id: int) -> int | None:
|
||||
"""Storage this user may still spend on ONE download: ``None`` for an unlimited user, else the
|
||||
cap minus the current footprint, floored at 0.
|
||||
|
||||
Zero is a real answer, not "no limit" — it means the cap is already met and the caller must
|
||||
REFUSE rather than proceed unbounded. `check_enqueue` cannot make that refusal on the caller's
|
||||
behalf: it ran when the job was queued, while the user was still under the cap, and the jobs
|
||||
that filled it have landed since. Keeping the arithmetic here (rather than inline in the worker)
|
||||
is what keeps every max_bytes decision in this module."""
|
||||
lim = resolve(db, user_id)
|
||||
if lim.unlimited:
|
||||
return None
|
||||
return max(0, lim.max_bytes - footprint(db, user_id))
|
||||
|
||||
|
||||
def check_upload(db: Session, user_id: int, add_bytes: int) -> None:
|
||||
"""Like check_enqueue, but for an upload whose size is KNOWN at ingest: also reject when adding
|
||||
this file would push the footprint over max_bytes. A download can't pre-check its size (unknown
|
||||
until the worker finishes) so check_enqueue only compares the current footprint; an upload can,
|
||||
so it's charged against the cap before it lands. Raises QuotaExceeded(max_jobs|max_bytes)."""
|
||||
lim = resolve(db, user_id)
|
||||
if lim.unlimited:
|
||||
return
|
||||
n = active_jobs(db, user_id)
|
||||
if n >= lim.max_jobs:
|
||||
raise QuotaExceeded("max_jobs", lim.max_jobs, n)
|
||||
projected = footprint(db, user_id) + add_bytes
|
||||
if projected > lim.max_bytes:
|
||||
raise QuotaExceeded("max_bytes", lim.max_bytes, projected)
|
||||
|
||||
|
||||
def usage(db: Session, user_id: int) -> dict:
|
||||
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
|
||||
lim = resolve(db, user_id)
|
||||
|
||||
@@ -11,21 +11,64 @@ writes the terminal job/asset state itself (done / error, or a cooperative pause
|
||||
progress hook), then exits 0. An unexpected crash exits non-zero and the parent fails the job.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from app import logs
|
||||
from app import logsink
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats
|
||||
from app.models import DownloadJob
|
||||
|
||||
log = logging.getLogger("siftlode.worker.runner")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def _deprioritize() -> None:
|
||||
"""Lower THIS child's CPU + IO priority so its yt-dlp/ffmpeg work — every descendant inherits the
|
||||
handicap — can never starve the co-located API, Postgres, or Plex playback. Run once at startup,
|
||||
before any heavy work. Every step is best-effort: a failed renice must never fail a download."""
|
||||
nice_inc = settings.download_child_nice
|
||||
if nice_inc > 0:
|
||||
try:
|
||||
os.nice(nice_inc) # relative increment; the child starts at the worker's nice (0)
|
||||
except OSError as exc:
|
||||
log.info("could not renice download child: %s", str(exc)[:120])
|
||||
cls = settings.download_child_ionice_class
|
||||
if cls > 0 and shutil.which("ionice"): # util-linux; absent on some minimal images
|
||||
try:
|
||||
subprocess.run(
|
||||
["ionice", "-c", str(cls), "-p", str(os.getpid())],
|
||||
capture_output=True,
|
||||
timeout=5,
|
||||
check=False,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError) as exc:
|
||||
log.info("could not ionice download child: %s", str(exc)[:120])
|
||||
# Cap OpenMP-parallel ffmpeg filters to the same budget as the encoder threads, so a filter
|
||||
# graph can't fan out across every core behind ffmpeg's own `-threads` cap.
|
||||
os.environ.setdefault("OMP_NUM_THREADS", str(editmod.encode_thread_cap()))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logs.configure_logging("runner")
|
||||
_deprioritize()
|
||||
if len(sys.argv) != 2:
|
||||
print("usage: python -m app.downloads.runner <job_id>", file=sys.stderr)
|
||||
# noqa T201: a usage line is for the person who typed the command. Logging is already
|
||||
# configured here, and in a deployment it writes JSON records — the one reader of this
|
||||
# line wants the bare sentence, not a record with a request id they do not have.
|
||||
print( # noqa: T201
|
||||
"usage: python -m app.downloads.runner <job_id>", file=sys.stderr
|
||||
)
|
||||
raise SystemExit(2)
|
||||
job_id = int(sys.argv[1])
|
||||
# This whole process exists to run ONE job — tag every line it prints with it, so the child's
|
||||
# output joins up with the worker's and the API's view of the same download.
|
||||
logs.set_job(job_id)
|
||||
logsink.start() # flushed at exit (atexit) — this process has no shutdown hook of its own
|
||||
|
||||
with SessionLocal() as db:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
@@ -35,17 +78,44 @@ def main() -> None:
|
||||
job_kind = job.job_kind
|
||||
spec = dict(job.profile_snapshot or {})
|
||||
source_kind, source_ref = job.source_kind, job.source_ref
|
||||
user_id = job.user_id
|
||||
if asset_id is None:
|
||||
return
|
||||
|
||||
_run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id)
|
||||
|
||||
|
||||
def _run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id) -> None:
|
||||
"""Run the job, and be the last stop before Python's own exception hook.
|
||||
|
||||
What gets past the pipeline is a failure it could not even record (the database refused the
|
||||
write that fails the job). Python's hook would print it to stderr raw — and its context is the
|
||||
download's own failure, whose text is the URL or ffmpeg's stderr naming the file. Here it is
|
||||
one line with the whole chain scrubbed; the non-zero exit then has the worker fail the job
|
||||
(`cause=child_exit`)."""
|
||||
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
||||
# real work, and to keep the import graph obvious: runner is a thin entry over the pipeline.
|
||||
from app.downloads.pipeline import _download, _process_edit
|
||||
|
||||
if job_kind == "edit":
|
||||
_process_edit(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec)
|
||||
with formats.hiding(source_ref):
|
||||
try:
|
||||
if job_kind == "edit":
|
||||
_process_edit(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
||||
# noqa TRY400: the stack goes with it scrubbed (`scrubbed_stack`); `exc_info` is raw.
|
||||
log.error( # noqa: TRY400
|
||||
"the download child for job %s failed",
|
||||
job_id,
|
||||
extra={
|
||||
**logs.event(
|
||||
"download.child.failed", job_id=job_id, error_type=type(exc).__name__
|
||||
),
|
||||
**logs.scrubbed_stack(exc, formats.log_safe),
|
||||
},
|
||||
)
|
||||
raise SystemExit(1) from None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -7,6 +7,7 @@ handles everything else. State transitions used by the routes (pause/resume/canc
|
||||
the worker live in `worker.py` / the routes; this module owns enqueue + asset bookkeeping.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
@@ -15,8 +16,9 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import adapters
|
||||
from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, quota, storage
|
||||
from app.downloads import formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobStatus
|
||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||
|
||||
@@ -79,11 +81,15 @@ def get_or_create_asset(
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
try:
|
||||
db.flush()
|
||||
# A SAVEPOINT: losing the race undoes only this insert, never the caller's own pending writes
|
||||
# (Retry clears the job's error before it gets here; a whole-session rollback threw that away).
|
||||
with db.begin_nested():
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
# silent-ok: the expected outcome of the race this savepoint exists for — two jobs wanting the
|
||||
# same asset. Losing it means the other insert won, and re-selecting is the whole handling.
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
asset = db.execute(stmt).scalar_one()
|
||||
return asset
|
||||
|
||||
@@ -155,7 +161,12 @@ def enqueue(
|
||||
Raises quota.QuotaExceeded if the user is at their job-count or footprint cap."""
|
||||
quota.check_enqueue(db, user_id)
|
||||
spec = formats.normalize(spec)
|
||||
sig = formats.format_sig(spec)
|
||||
# If this user has a stored cookie for the download's host, salt the cache signature with their
|
||||
# id so the cookie-authed result is a PRIVATE asset — never dedup-shared with, or cache-served
|
||||
# to, another user (the load-bearing privacy fix; the worker attaches the cookie to exactly this
|
||||
# salted identity). A normal download (no matching cookie) gets salt=None and shares as before.
|
||||
salt = credentials.salt_for_enqueue(db, user_id, source_url(source_kind, source_ref))
|
||||
sig = formats.format_sig(spec, salt=salt)
|
||||
asset = get_or_create_asset(db, source_kind, source_ref, sig)
|
||||
# Show a real title + thumbnail the moment it's queued (feed downloads are catalog videos).
|
||||
populate_from_catalog(db, asset)
|
||||
@@ -210,6 +221,10 @@ def enqueue_edit(
|
||||
src_asset = db.get(MediaAsset, source_job.asset_id) if source_job.asset_id else None
|
||||
if src_asset is None or src_asset.status != "ready" or not src_asset.rel_path:
|
||||
raise EditError("source_not_ready")
|
||||
# A generic uploaded file (media_kind='file') has no timeline to trim/crop — the editor path is
|
||||
# media-only. Reject here so an upload's file never reaches the ffmpeg edit worker.
|
||||
if src_asset.media_kind == mediakind.FILE:
|
||||
raise EditError("not_media")
|
||||
|
||||
spec = editmod.normalize_edit_spec(edit_spec)
|
||||
if not spec:
|
||||
@@ -226,6 +241,9 @@ def enqueue_edit(
|
||||
asset.thumbnail_url = src_asset.thumbnail_url
|
||||
asset.upload_date = src_asset.upload_date
|
||||
asset.duration_s = editmod.clip_duration(spec, src_asset.duration_s)
|
||||
# A clip inherits its source's kind (a video edit stays video, an audio trim stays audio);
|
||||
# the ffmpeg edit worker doesn't re-probe codecs, so set it here where the source is known.
|
||||
asset.media_kind = src_asset.media_kind or mediakind.VIDEO
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
@@ -328,13 +346,20 @@ def pause(db: Session, job: DownloadJob) -> None:
|
||||
def resume(db: Session, job: DownloadJob) -> None:
|
||||
"""Requeue a paused/errored job for a fresh attempt, clearing its error. If the SHARED asset itself
|
||||
failed, reset it to pending so the worker actually re-downloads — otherwise the requeued job would
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset)."""
|
||||
instantly re-inherit the asset's stale error (the worker short-circuits an errored asset).
|
||||
|
||||
A cookie the user added AFTER the job was queued moves the job onto their private asset first
|
||||
(`_follow_the_cookie`): the worker only ever attaches a cookie to that identity, so without the move
|
||||
the Retry a login-wall message asks for would fail on the same wall."""
|
||||
if job.status in (JobStatus.PAUSED, JobStatus.ERROR):
|
||||
job.progress = 0
|
||||
job.error = None
|
||||
job.error_code = None
|
||||
job.error_detail = None
|
||||
if job.asset_id:
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
if asset is not None:
|
||||
asset = _follow_the_cookie(db, job, asset)
|
||||
if asset and asset.status == "error": # MediaAsset.status (no enum), not a JobStatus
|
||||
asset.status = "pending"
|
||||
asset.error = None
|
||||
@@ -342,6 +367,32 @@ def resume(db: Session, job: DownloadJob) -> None:
|
||||
db.commit()
|
||||
|
||||
|
||||
def _follow_the_cookie(db: Session, job: DownloadJob, asset: MediaAsset) -> MediaAsset:
|
||||
"""Re-point a requeued job at the asset enqueue would pick NOW, when the user has a cookie for the
|
||||
link's host but the job still holds an asset the worker won't attach it to (queued before the
|
||||
cookie existed). Returns the asset the job holds afterwards.
|
||||
|
||||
Only a `pending` / `error` asset moves — neither has a file nor a download in flight. A `ready` one
|
||||
already has the file, and a `downloading` one may be this job's own child still being stopped by
|
||||
the pause; the next Retry after it settles moves the job then. Removing a cookie never moves a job
|
||||
back: the worker attaches nothing without a credential, so the private asset is as good as any."""
|
||||
if asset.status not in ("pending", "error"):
|
||||
return asset
|
||||
url = source_url(job.source_kind, job.source_ref)
|
||||
salt = credentials.salt_for_enqueue(db, job.user_id, url)
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
if salt is None or credentials.is_authed_asset(asset, spec, job.user_id, url):
|
||||
return asset
|
||||
target = get_or_create_asset(
|
||||
db, job.source_kind, job.source_ref, formats.format_sig(spec, salt=salt)
|
||||
)
|
||||
populate_from_catalog(db, target)
|
||||
release_asset(db, job) # drop the old hold BEFORE re-pointing — it keys off job.asset_id
|
||||
incr_ref_count(db, target.id)
|
||||
job.asset_id = target.id
|
||||
return target
|
||||
|
||||
|
||||
def cancel(db: Session, job: DownloadJob) -> None:
|
||||
"""Cancel a job unless it's already done/canceled, releasing its asset hold WHILE it still counts
|
||||
as holding — `canceled` ∉ HOLDING, so a later delete won't double-release."""
|
||||
@@ -360,13 +411,31 @@ def delete(db: Session, job: DownloadJob) -> None:
|
||||
|
||||
def redownload(db: Session, job: DownloadJob) -> None:
|
||||
"""Rebuild a finished/failed job's file into a FRESH, private asset (the cache sig is salted with
|
||||
the job id + current asset id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
the new asset's own id, so it never reuses the dedup pool) and requeue the SAME job row —
|
||||
so any public share link (DownloadLink -> job_id) keeps resolving to the newly-downloaded file.
|
||||
The old hold is dropped first (freeing the old file iff this job was its last holder). The caller
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409)."""
|
||||
has already rejected edit clips + still-in-progress jobs (HTTP 409).
|
||||
|
||||
When the user has a cookie for the link's host, the salt carries it (`credentials.rebuild_salt`),
|
||||
so the worker recognises the rebuild as their private identity and attaches the cookie — a cookie
|
||||
download, or one the user added after a failure, rebuilds with the cookie."""
|
||||
spec = formats.normalize(job.profile_snapshot)
|
||||
sig = formats.format_sig(spec, salt=f"{job.id}:{job.asset_id}")
|
||||
asset = get_or_create_asset(db, job.source_kind, job.source_ref, sig)
|
||||
cookie = credentials.salt_for_enqueue(
|
||||
db, job.user_id, source_url(job.source_kind, job.source_ref)
|
||||
)
|
||||
# The salt names the row's own id, which exists only once the row does: insert it under a unique
|
||||
# placeholder, then write the real signature in the same transaction (nobody sees the placeholder).
|
||||
asset = MediaAsset(
|
||||
source_kind=job.source_kind,
|
||||
source_ref=job.source_ref,
|
||||
format_sig=f"rebuild:{uuid.uuid4().hex}",
|
||||
status="pending",
|
||||
ref_count=0,
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush()
|
||||
asset.format_sig = formats.format_sig(spec, salt=credentials.rebuild_salt(asset.id, cookie))
|
||||
db.flush()
|
||||
populate_from_catalog(db, asset)
|
||||
# Drop the old hold BEFORE re-pointing the job — release_asset keys off the job's current asset.
|
||||
release_asset(db, job)
|
||||
@@ -376,6 +445,8 @@ def redownload(db: Session, job: DownloadJob) -> None:
|
||||
job.progress = 0
|
||||
job.phase = None
|
||||
job.error = None
|
||||
job.error_code = None # a rebuild that fails again gets its own diagnosis, never the old one
|
||||
job.error_detail = None
|
||||
job.speed_bps = None
|
||||
job.eta_s = None
|
||||
# Re-derive the display name from the freshly cleaned title on completion (else an older, un-cleaned
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
"""The failures the download pipeline raises itself, with the whole story in the message.
|
||||
|
||||
`ffmpeg failed (1): …`, `yt-dlp produced no output file`, a post that shows no video: the message
|
||||
says what happened, and the stack is only the `except` that caught it. A failure line leaves that
|
||||
stack off (`pipeline._failure_stack`) — but ONLY for these. A plain `RuntimeError` is also what
|
||||
Python raises for a bug of ours (a dict changed size during iteration, a generator's
|
||||
StopIteration, `RecursionError`, `NotImplementedError`), and that one's stack is the only lead.
|
||||
"""
|
||||
|
||||
|
||||
class StatedFailure(RuntimeError):
|
||||
"""A failure we raised on purpose, whose message is its full explanation. A `RuntimeError` so
|
||||
every `except RuntimeError` that caught these before still does."""
|
||||
@@ -19,6 +19,11 @@ from datetime import date
|
||||
from pathlib import Path
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
from app import logs
|
||||
from app.downloads import mediakind
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Reserve room for the "_id.ext" suffix + parent dirs within the 255-char component limit.
|
||||
_MAX_TITLE = 120
|
||||
# User-facing (Content-Disposition) names can be longer than an on-disk component.
|
||||
@@ -147,6 +152,81 @@ def edit_rel_path(user_id: int, asset_id: int, ext: str) -> str:
|
||||
return f".edits/{user_id}/clip_{asset_id}.{ext}"
|
||||
|
||||
|
||||
def _clean_upload_ext(filename: str) -> str:
|
||||
"""The upload's extension reduced to a shell-safe, lowercased, bounded token (``""`` when the
|
||||
upload has no extension). The SINGLE definition feeding both the on-disk suffix and the stored
|
||||
`container` — they must agree (the served Content-Disposition suffix has to match the container),
|
||||
so the sanitize/truncation rule lives in one place, not two drifting copies."""
|
||||
return _UNSAFE.sub("", Path(filename).suffix.lstrip(".")).lower()[:16]
|
||||
|
||||
|
||||
def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
|
||||
"""On-disk path (relative to DOWNLOAD_ROOT) for a locally-uploaded file.
|
||||
|
||||
Kept in a per-user `.uploads/` tree — device-downloadable via the file endpoint, but NOT part of
|
||||
the Plex library (an upload is arbitrary user content, not a canonical episode; no .nfo/poster
|
||||
episode sidecars). The name is system-decided and bound to the asset id (`_a{id}` keeps two
|
||||
same-named uploads apart); the original filename is display metadata only. The extension is
|
||||
preserved (shell-safe, bounded) so the served file + its Content-Disposition carry a correct
|
||||
suffix; an extensionless upload stays extensionless."""
|
||||
stem = sanitize(Path(filename).stem, 80)
|
||||
ext = _clean_upload_ext(filename)
|
||||
suffix = f".{ext}" if ext else ""
|
||||
return f".uploads/{user_id}/{stem}_a{asset_id}{suffix}"
|
||||
|
||||
|
||||
def upload_ext(filename: str) -> str | None:
|
||||
"""The stored `container` for an uploaded file: its shell-safe lowercased extension, or None when
|
||||
the upload has no extension (a bare name). Shares `_clean_upload_ext` with `upload_rel_path` so the
|
||||
stored container and the on-disk suffix can never diverge."""
|
||||
return _clean_upload_ext(filename) or None
|
||||
|
||||
|
||||
# The Content-Type used to serve an inline-playable media asset, keyed by its stored `container`.
|
||||
# DELIBERATELY a strict allow-list of known audio/video containers: the served type is chosen from
|
||||
# the asset's ffprobe-verified media_kind + this table, NEVER guessed from the raw (user-controlled)
|
||||
# upload extension. Without this, an uploaded media/HTML polyglot named "evil.html" — a real MP3/MP4
|
||||
# that ffprobe classifies as media, so it dodges the generic-file force-download — would be served
|
||||
# with a Content-Type Starlette guesses from ".html" (text/html) and execute as stored XSS on the
|
||||
# serving origin. A media asset whose container isn't in this table falls back to a neutral
|
||||
# force-download attachment (see `served_media_type`), so nothing script-executable is ever inline.
|
||||
_MEDIA_MIME: dict[str, str] = {
|
||||
# video containers
|
||||
"mp4": "video/mp4",
|
||||
"m4v": "video/mp4",
|
||||
"mov": "video/quicktime",
|
||||
"webm": "video/webm",
|
||||
"mkv": "video/x-matroska",
|
||||
"ogv": "video/ogg",
|
||||
"avi": "video/x-msvideo",
|
||||
# audio containers
|
||||
"m4a": "audio/mp4",
|
||||
"mp3": "audio/mpeg",
|
||||
"aac": "audio/aac",
|
||||
"ogg": "audio/ogg",
|
||||
"oga": "audio/ogg",
|
||||
"opus": "audio/ogg",
|
||||
"weba": "audio/webm",
|
||||
"wav": "audio/wav",
|
||||
"flac": "audio/flac",
|
||||
}
|
||||
|
||||
|
||||
def served_media_type(media_kind: str | None, container: str | None) -> str | None:
|
||||
"""The Content-Type a library file MUST be served with — a real audio/video MIME for a recognised
|
||||
media container, else ``None`` meaning "serve as a neutral force-download attachment".
|
||||
|
||||
Returns None for a generic ``file`` kind AND for any media whose container we don't positively
|
||||
recognise as safe. Callers serve None as ``application/octet-stream`` + ``attachment`` so the
|
||||
served Content-Type is NEVER derived from the (attacker-controlled) file extension — the guard
|
||||
that stops uploaded HTML/SVG or a media/HTML polyglot from executing inline on the serving
|
||||
origin. A ``None`` media_kind (a legacy asset before the discriminator was backfilled) is treated
|
||||
as media so an old download keeps its inline player."""
|
||||
if media_kind == mediakind.FILE:
|
||||
return None
|
||||
return _MEDIA_MIME.get((container or "").lower())
|
||||
|
||||
|
||||
def download_filename(display_name: str, container: str | None, path: Path) -> str:
|
||||
"""The Content-Disposition filename for a served download: the cleaned display name carrying a single
|
||||
correct extension — the asset container, else the file's own suffix — without doubling it when the
|
||||
@@ -217,7 +297,13 @@ def write_sidecars(download_root: str, rel: str, meta: MediaMeta, thumb_src: Pat
|
||||
if thumb_src and thumb_src.exists():
|
||||
shutil.copyfile(str(thumb_src), str(_sidecar(media, ".jpg")))
|
||||
return True
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# The download itself still completes, so the job reads as done while the library it was
|
||||
# downloaded FOR has no .nfo to read and no poster to show.
|
||||
log.warning(
|
||||
"the sidecars beside a download could not be written",
|
||||
extra=logs.event("download.sidecars.write_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@@ -227,14 +313,23 @@ def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
except FileNotFoundError:
|
||||
pass # silent-ok: already gone is the outcome this function wanted
|
||||
except OSError as exc:
|
||||
# The row goes and the bytes stay. Nothing else ever revisits this path, so the disk
|
||||
# quietly keeps a file no account has any way to see, let alone delete.
|
||||
log.info(
|
||||
"a deleted download left a file behind",
|
||||
extra=logs.event("download.asset.file_unlink_failed", reason=type(exc).__name__),
|
||||
)
|
||||
# Prune empty Season/Channel dirs up to (but not including) the root.
|
||||
root = Path(download_root).resolve()
|
||||
parent = media.parent
|
||||
while parent != root and root in parent.parents:
|
||||
try:
|
||||
parent.rmdir() # only succeeds if empty
|
||||
# silent-ok: this `rmdir` failing is the loop's stop condition — the parent still
|
||||
# holds another download's files, so the prune has gone as far up as it may.
|
||||
except OSError:
|
||||
break
|
||||
parent = parent.parent
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Local file ingest for the download center (E7 S5) — the non-yt-dlp path into the media library.
|
||||
|
||||
A user-uploaded file joins the SAME library as a downloaded one, so it inherits the whole function
|
||||
set already built (metadata chips, share links, and — for media — in-app play/edit). Unlike a
|
||||
download it bypasses yt-dlp AND the worker entirely: the file is already on the host, so ingest
|
||||
probes it, places it on disk, and creates an already-`ready` MediaAsset + `done` DownloadJob in one
|
||||
step — the same "the file exists, mark it done now" shortcut `service.enqueue` takes on a cache hit.
|
||||
|
||||
Kinds (the invariant lives in `mediakind`): a probe with a real video stream -> video, audio-only
|
||||
-> audio, neither -> a generic 'file' (pdf/zip/docx/…) that is download-only and never rendered
|
||||
inline (see the force-download serving in the file/watch routes). This module is the ingest seam a
|
||||
future sub-function (e.g. convert) plugs a new adapter into, mirroring how it sidesteps the worker.
|
||||
|
||||
An upload does NOT get an `expires_at`: it's user-provided and may be irreplaceable (there's no
|
||||
source to re-download), so it doesn't age out on the retention clock the way a re-fetchable download
|
||||
does — it lives until the user deletes it.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, probe, quota, service, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
|
||||
|
||||
def _title_from_filename(filename: str) -> str:
|
||||
"""A human display title from the original upload name: the stem with separators tidied to
|
||||
spaces (kept as-is otherwise — this is display metadata, not the shell-safe on-disk name)."""
|
||||
stem = Path(filename).stem.replace("_", " ").replace(".", " ").strip()
|
||||
return (stem or "Upload")[:255]
|
||||
|
||||
|
||||
def ingest_upload(
|
||||
db: Session,
|
||||
user_id: int,
|
||||
*,
|
||||
filename: str,
|
||||
src_path: Path,
|
||||
size: int,
|
||||
display_name: str | None = None,
|
||||
) -> DownloadJob:
|
||||
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return the `done`
|
||||
job. `src_path` is MOVED into place on success (the caller stages it to a temp file first).
|
||||
|
||||
Raises quota.QuotaExceeded (holdings/footprint cap) — checked BEFORE any disk write, so a
|
||||
rejected upload leaves nothing behind and the caller just cleans up its temp file."""
|
||||
quota.check_upload(db, user_id, size) # the file's size is known → charge it up front
|
||||
|
||||
vcodec, acodec, width, height, duration = probe.probe_media(src_path)
|
||||
kind = mediakind.derive(vcodec, acodec)
|
||||
|
||||
# An upload never dedups (each file is its own asset): a random ref makes the cache key unique,
|
||||
# so `format_sig` is a constant marker rather than a format hash.
|
||||
ref = f"{user_id}:{secrets.token_hex(8)}"[:512]
|
||||
asset = MediaAsset(
|
||||
source_kind="upload", source_ref=ref, format_sig="upload", status="pending", ref_count=0
|
||||
)
|
||||
db.add(asset)
|
||||
db.flush() # need asset.id for the on-disk path
|
||||
|
||||
rel = storage.upload_rel_path(user_id, asset.id, filename)
|
||||
storage.place_file(src_path, settings.download_root, rel)
|
||||
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
|
||||
# asset/job rows roll back but the placed file would be ORPHANED — nothing references it, the
|
||||
# per-user delete path keys off asset.rel_path (which is gone), and the GC orphan sweep skips the
|
||||
# dot-prefixed `.uploads/` tree. So roll back the on-disk side too on any failure.
|
||||
try:
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
actual_size = final.stat().st_size if final.exists() else size
|
||||
|
||||
asset.status = "ready"
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = actual_size
|
||||
asset.container = storage.upload_ext(filename)
|
||||
asset.vcodec = vcodec
|
||||
asset.acodec = acodec
|
||||
asset.width = width
|
||||
asset.height = height
|
||||
asset.duration_s = duration
|
||||
asset.media_kind = kind
|
||||
asset.title = (display_name or "").strip() or _title_from_filename(filename)
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
# No expires_at — see the module docstring (an upload is irreplaceable, so it doesn't age out).
|
||||
if kind == mediakind.VIDEO:
|
||||
# A frame poster gives the library row a thumbnail + the share link an og:image. Audio has
|
||||
# no frame and a generic file no player, so neither gets one. Best-effort (ffmpeg decodes
|
||||
# server side even for a codec the browser can't play).
|
||||
asset.poster_path = editmod.ensure_poster(asset, settings.download_root)
|
||||
|
||||
job = DownloadJob(
|
||||
user_id=user_id,
|
||||
asset_id=asset.id,
|
||||
job_kind="upload",
|
||||
source_kind="upload",
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
display_name=(display_name or "").strip() or None,
|
||||
status="done",
|
||||
progress=100,
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
service.incr_ref_count(db, asset.id)
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
# The rows are gone; remove the now-unreferenced file so no bytes leak. Clean the poster by its
|
||||
# DETERMINISTIC path (rel with a .jpg suffix, the same path ensure_poster writes to) rather than
|
||||
# ensure_poster's return value: it writes the frame directly to that path and returns None on an
|
||||
# ffmpeg failure, so a partial poster can be on disk while the return is None — gating on the
|
||||
# return would skip that orphan. unlink is missing_ok, so the poster path is a harmless no-op
|
||||
# when none was generated (audio/file, or a video whose poster never started).
|
||||
for leaked in (rel, str(Path(rel).with_suffix(".jpg"))):
|
||||
storage.abs_path(settings.download_root, leaked).unlink(missing_ok=True)
|
||||
raise
|
||||
db.refresh(job)
|
||||
return job
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Refuse a user-supplied download URL that points back into our own network (SEC-12).
|
||||
|
||||
The Download Center deliberately accepts an arbitrary ``http(s)`` URL — that is the feature — and
|
||||
hands it to yt-dlp running on the WORKER. Without a guard, any approved user can make the server
|
||||
fetch anything the server can reach: other containers on the compose network, the LAN, a cloud
|
||||
metadata endpoint. The failure text then comes back to them in ``job.error``, so "connection
|
||||
refused" vs "HTTP Error 401" vs "timed out" turns the download box into an internal port scanner.
|
||||
|
||||
What this closes: a URL whose host resolves to a loopback / private / link-local / reserved address
|
||||
is refused, at enqueue AND again in the worker before extraction. Both, because DNS can answer
|
||||
differently between the two moments, and the worker is where the socket is actually opened.
|
||||
|
||||
What it does NOT close, stated plainly rather than implied, because both need the same fix — gating
|
||||
the connection itself with a custom socket factory inside yt-dlp, a much larger change than this:
|
||||
|
||||
* DNS rebinding: a host that passes the worker's check and then answers yt-dlp's own resolution
|
||||
with an internal address.
|
||||
* HTTP redirects: a public host that answers 3xx with an internal ``Location``. yt-dlp follows it,
|
||||
and only the URL the user TYPED was ever judged here. An attacker-controlled host or any public
|
||||
open redirect reaches it, so this is the cheaper of the two. It does NOT usually yield the
|
||||
response body — a non-media body fails the mp4 remux and stores nothing — but it does hand back
|
||||
the readout above: the redaction in ``errors.user_error_text`` fires only for the ``network``
|
||||
class, so a dead internal port (refused → ``network`` → redacted) is distinguishable from a live
|
||||
one (an HTTP answer that is not media → ``unsupported_host``/``format_unavailable``, raw text
|
||||
kept). Tracked as SEC-17.
|
||||
|
||||
The two checks here stop the straightforward case (typing ``http://127.0.0.1:6379`` or an internal
|
||||
hostname into the box), which is the reachable one.
|
||||
|
||||
Not used for YouTube jobs: those resolve to a fixed public host from an 11-char id, so there is no
|
||||
user-controlled destination to guard.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Reason codes travel to the client inside the structured 422 detail, so they are part of the API
|
||||
# vocabulary (frontend `localizeDetail`) — keep them stable.
|
||||
REASON_SCHEME = "scheme" # not http(s)
|
||||
REASON_NO_HOST = "no_host" # URL carries no hostname
|
||||
REASON_MALFORMED = "malformed" # unparseable authority or port; urlparse raises late, so guard it
|
||||
REASON_UNRESOLVABLE = "unresolvable" # DNS said nothing (honest: not the same as "blocked")
|
||||
REASON_PRIVATE = "private" # resolves into a network the server must not fetch from
|
||||
|
||||
|
||||
class BlockedUrl(Exception):
|
||||
"""A user-supplied URL that must not be fetched server-side. ``reason`` is one of the
|
||||
REASON_* codes above."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
def _is_forbidden(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
||||
"""Every address family we must not open a connection to. Checked explicitly rather than
|
||||
leaning on ``is_private`` alone, whose coverage differs between IPv4 and IPv6."""
|
||||
# ::ffff:127.0.0.1 is loopback wearing an IPv6 hat — unwrap before judging it.
|
||||
mapped = getattr(ip, "ipv4_mapped", None)
|
||||
if mapped is not None:
|
||||
ip = mapped
|
||||
return bool(
|
||||
ip.is_private
|
||||
or ip.is_loopback
|
||||
or ip.is_link_local
|
||||
or ip.is_multicast
|
||||
or ip.is_reserved
|
||||
or ip.is_unspecified
|
||||
)
|
||||
|
||||
|
||||
def _resolve(host: str, port: int | None) -> list[str]:
|
||||
"""Every address ``host`` resolves to. Raises BlockedUrl(REASON_UNRESOLVABLE) when DNS has no
|
||||
answer — reported as its own outcome, because "we could not look that up" and "we refuse to
|
||||
fetch that" are different things to tell a user."""
|
||||
try:
|
||||
infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
|
||||
except socket.gaierror as exc:
|
||||
raise BlockedUrl(REASON_UNRESOLVABLE) from exc
|
||||
return [info[4][0] for info in infos]
|
||||
|
||||
|
||||
def check_url(raw: str) -> None:
|
||||
"""Raise BlockedUrl unless ``raw`` is an http(s) URL whose host resolves only to addresses the
|
||||
server is allowed to fetch from. A host with even ONE forbidden address is refused: a
|
||||
round-robin answer must not become a coin flip."""
|
||||
# `urlparse` itself raises ValueError on a malformed authority ("http://[::1/x" — an unclosed
|
||||
# bracket), and `.port` raises on ":99999" / ":abc" when READ rather than at parse time. Both
|
||||
# escape the caller's `except BlockedUrl` as a 500 unless caught here, which is the opposite of
|
||||
# what this module exists to do: turn a bad URL into a structured, localizable refusal.
|
||||
try:
|
||||
u = urlparse((raw or "").strip())
|
||||
port = u.port
|
||||
except ValueError as exc:
|
||||
raise BlockedUrl(REASON_MALFORMED) from exc
|
||||
if u.scheme not in ("http", "https"):
|
||||
raise BlockedUrl(REASON_SCHEME)
|
||||
host = u.hostname
|
||||
if not host:
|
||||
raise BlockedUrl(REASON_NO_HOST)
|
||||
for addr in _resolve(host, port):
|
||||
try:
|
||||
ip = ipaddress.ip_address(addr)
|
||||
except ValueError: # getaddrinfo returned something unparseable; treat as unusable
|
||||
raise BlockedUrl(REASON_PRIVATE) from None
|
||||
if _is_forbidden(ip):
|
||||
raise BlockedUrl(REASON_PRIVATE)
|
||||
+264
-118
@@ -3,36 +3,101 @@
|
||||
Gmail SMTP + App Password by default. Deliberately fail-soft: if SMTP isn't configured
|
||||
or a send errors, we log and return False so the caller can fall back to in-app
|
||||
notifications — email is never load-bearing for the app's core flow.
|
||||
|
||||
Fail-soft must not mean invisible, though: every drop also leaves an in-app notice in the admins'
|
||||
inbox (see `_alert_admins`). An instance whose SMTP was never filled in silently swallows
|
||||
verification links, password resets and access-request notices, and the operator's first clue used
|
||||
to be a person telling them nothing arrived.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import html
|
||||
import re
|
||||
import smtplib
|
||||
import ssl
|
||||
from email.message import EmailMessage
|
||||
from email.utils import formatdate
|
||||
|
||||
from app import sysconfig
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, mailtext, sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import User
|
||||
from app.notifications import create_notification
|
||||
from app.ratelimit import RateLimiter
|
||||
|
||||
log = logging.getLogger("siftlode.email")
|
||||
log = logs.get_logger(__name__)
|
||||
# One admin notice an hour per reason. A relay that is down fails every send, and the inbox must
|
||||
# stay readable; the log keeps the full record either way.
|
||||
_alert_throttle = RateLimiter(max_events=1, window_seconds=3600)
|
||||
|
||||
|
||||
def _smtp() -> dict:
|
||||
"""Effective SMTP config (admin DB override over env defaults). Opens its own short
|
||||
session because email is often sent from a BackgroundTask, outside a request's db."""
|
||||
with SessionLocal() as db:
|
||||
return {
|
||||
"host": sysconfig.get_str(db, "smtp_host"),
|
||||
"port": sysconfig.get_int(db, "smtp_port"),
|
||||
"user": sysconfig.get_str(db, "smtp_user"),
|
||||
"password": sysconfig.get_str(db, "smtp_password"),
|
||||
"from": sysconfig.get_str(db, "smtp_from"),
|
||||
}
|
||||
def _alert_admins(reason: str, subject: str, recipients: list[str]) -> None:
|
||||
"""Record an in-app notice that a message did NOT go out. Best effort in every direction: it
|
||||
opens its own session (callers are often background tasks), swallows its own errors, and never
|
||||
reports failure to the caller — this is the fallback path, it must not become a new way to
|
||||
break a send."""
|
||||
if not _alert_throttle.allow(reason):
|
||||
return
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
admins = (
|
||||
db.execute(
|
||||
select(User.id).where(
|
||||
User.role == "admin",
|
||||
User.is_active.is_(True),
|
||||
User.is_suspended.is_(False),
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not admins:
|
||||
# Nothing was recorded, so don't let this cost the next hour of alerts on the very
|
||||
# path whose job is to stop drops going unseen. Same on the failure branch below.
|
||||
_alert_throttle.reset()
|
||||
return
|
||||
for uid in admins:
|
||||
create_notification(
|
||||
db,
|
||||
user_id=uid,
|
||||
type="email_undelivered",
|
||||
# English fallback; the inbox renders a translated version from `data`.
|
||||
title="Email could not be sent",
|
||||
body=subject,
|
||||
data={"reason": reason, "subject": subject, "recipients": recipients},
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
except Exception:
|
||||
_alert_throttle.reset()
|
||||
log.exception("Could not record the undelivered-email notice (%r)", subject)
|
||||
|
||||
|
||||
def email_enabled() -> bool:
|
||||
c = _smtp()
|
||||
def _read_smtp(db: Session) -> dict:
|
||||
return {
|
||||
"host": sysconfig.get_str(db, "smtp_host"),
|
||||
"port": sysconfig.get_int(db, "smtp_port"),
|
||||
"user": sysconfig.get_str(db, "smtp_user"),
|
||||
"password": sysconfig.get_str(db, "smtp_password"),
|
||||
"from": sysconfig.get_str(db, "smtp_from"),
|
||||
}
|
||||
|
||||
|
||||
def _smtp(db: Session | None = None) -> dict:
|
||||
"""Effective SMTP config (admin DB override over env defaults). Opens its own short session when
|
||||
the caller has none, because email is usually sent from a BackgroundTask outside a request's db.
|
||||
A caller that IS inside a request should pass its session: reading the password decrypts a Fernet
|
||||
value, and /api/me polls this for every admin tab."""
|
||||
if db is not None:
|
||||
return _read_smtp(db)
|
||||
with SessionLocal() as own:
|
||||
return _read_smtp(own)
|
||||
|
||||
|
||||
def email_enabled(db: Session | None = None) -> bool:
|
||||
c = _smtp(db)
|
||||
return bool(c["host"] and c["user"] and c["password"])
|
||||
|
||||
|
||||
@@ -40,13 +105,64 @@ def _admin_contact() -> str | None:
|
||||
return next(iter(sorted(settings.admin_email_set)), None)
|
||||
|
||||
|
||||
# A bare URL in a text/plain mail is only clickable if the CLIENT decides to linkify it. Gmail,
|
||||
# Outlook and Apple Mail do; plenty of lighter web clients render the body verbatim and leave the
|
||||
# link as dead text — which, for a verification or password-reset mail, is the whole message. So
|
||||
# every mail also carries an HTML alternative built from the SAME plain body (below), rather than
|
||||
# per-template HTML that would have to be kept in sync with it.
|
||||
#
|
||||
# `[^\s<]` stops at whitespace and at the `<` of any following markup; the trailing-punctuation trim
|
||||
# keeps a sentence-final period out of the href. `;` and `:` are deliberately NOT trimmed — escaping
|
||||
# turns `&` into `&`, so a query string can legitimately end in a semicolon.
|
||||
_URL_RE = re.compile(r"https?://[^\s<]+")
|
||||
|
||||
|
||||
def _html_body(text: str) -> str:
|
||||
"""Render a plain-text mail body as minimal HTML: escaped, blank-line-separated paragraphs, with
|
||||
real links. Escaping happens FIRST and the linkifier then runs over the escaped text — bodies
|
||||
interpolate untrusted values (a requester's email address, the operator contact), so no caller
|
||||
can inject markup, and `&` inside a URL is already `&` by the time it reaches the href, which
|
||||
is what an HTML attribute wants anyway."""
|
||||
|
||||
def _anchor(m: re.Match[str]) -> str:
|
||||
url = m.group(0)
|
||||
trail = ""
|
||||
while url and url[-1] in ".,!?)":
|
||||
url, trail = url[:-1], url[-1] + trail
|
||||
if not url:
|
||||
return trail
|
||||
return f'<a href="{url}">{url}</a>{trail}'
|
||||
|
||||
paragraphs = [p for p in html.escape(text).split("\n\n") if p.strip()]
|
||||
rendered = "".join(
|
||||
f"<p>{_URL_RE.sub(_anchor, p).replace(chr(10), '<br>')}</p>" for p in paragraphs
|
||||
)
|
||||
# Inline styles only — many clients drop <style> blocks. No images, no external CSS, no tracking:
|
||||
# the HTML part exists to make the link clickable, not to become a template to maintain.
|
||||
return (
|
||||
'<html><body style="margin:0;padding:16px;font-family:system-ui,-apple-system,'
|
||||
"'Segoe UI',sans-serif;font-size:15px;line-height:1.6\">"
|
||||
f'<div style="max-width:560px">{rendered}</div>'
|
||||
"</body></html>"
|
||||
)
|
||||
|
||||
|
||||
def _send(to: list[str], subject: str, body: str, reply_to: str | None = None) -> bool:
|
||||
recipients = [e for e in to if e]
|
||||
if not recipients:
|
||||
return False
|
||||
c = _smtp()
|
||||
if not (c["host"] and c["user"] and c["password"]):
|
||||
log.info("SMTP not configured; skipping email %r to %s", subject, recipients)
|
||||
# The count, not the addresses: `redact_text` rewrites every one of them to `[email]`
|
||||
# before either sink sees the line, so the list only ever printed as noise. The subject
|
||||
# stays — the templates are static, and the one that interpolates (`admin_new_request`)
|
||||
# interpolates an address, which the same redactor covers.
|
||||
log.info(
|
||||
"SMTP not configured; skipping email %r to %d recipient(s)",
|
||||
subject,
|
||||
len(recipients), # log-pii-ok: how many, not who
|
||||
)
|
||||
_alert_admins("not_configured", subject, recipients)
|
||||
return False
|
||||
msg = EmailMessage()
|
||||
msg["Subject"] = subject
|
||||
@@ -57,138 +173,168 @@ def _send(to: list[str], subject: str, body: str, reply_to: str | None = None) -
|
||||
msg["Date"] = formatdate(localtime=True)
|
||||
if reply_to:
|
||||
msg["Reply-To"] = reply_to
|
||||
# text/plain stays the primary part (it is the source of truth, and a text-only client still
|
||||
# gets the full message); the HTML alternative below is what makes the link clickable everywhere.
|
||||
msg.set_content(body)
|
||||
msg.add_alternative(_html_body(body), subtype="html")
|
||||
try:
|
||||
with smtplib.SMTP(c["host"], c["port"], timeout=20) as s:
|
||||
s.starttls(context=ssl.create_default_context())
|
||||
s.login(c["user"], c["password"])
|
||||
s.send_message(msg)
|
||||
log.info("Sent email %r to %s", subject, recipients)
|
||||
# log-pii-ok: how many, not who
|
||||
log.info("Sent email %r to %d recipient(s)", subject, len(recipients))
|
||||
return True
|
||||
except Exception:
|
||||
log.exception("SMTP send failed (%r to %s)", subject, recipients)
|
||||
# log-pii-ok: how many, not who
|
||||
log.exception("SMTP send failed (%r to %d recipient(s))", subject, len(recipients))
|
||||
_alert_admins("send_failed", subject, recipients)
|
||||
return False
|
||||
|
||||
|
||||
def send_access_approved(email: str) -> bool:
|
||||
def _next_step(verify_link: str | None, has_account: bool, hours: int, lang: str) -> str:
|
||||
"""What this person actually has to do next, in the one paragraph that carries the whole flow.
|
||||
|
||||
Three states, because a mail that guesses wrong sends someone looking for a link that isn't
|
||||
coming — or worse, tells them to sign in when their address is still unconfirmed and sign-in will
|
||||
refuse them (E9.5 S4 named that state on screen; the mail must not contradict it).
|
||||
|
||||
The link branch is the point of the C option: in the common ordering — approve first, the person
|
||||
signs up later, or signs up and waits — they would otherwise have to find and open TWO separate
|
||||
messages. One mail, one click.
|
||||
"""
|
||||
if verify_link:
|
||||
return mailtext.text(
|
||||
"next_step.verify", lang, link=verify_link, expiry=mailtext.link_expiry(hours, lang)
|
||||
)
|
||||
if has_account:
|
||||
return mailtext.text("next_step.sign_in", lang, app=settings.app_base)
|
||||
return mailtext.text("next_step.register", lang, app=settings.app_base)
|
||||
|
||||
|
||||
def _reply_contact(lang: str) -> tuple[str | None, str]:
|
||||
"""The admin address to answer to, and the closing line that names it — empty when no admin
|
||||
contact is configured, so the mail never promises a reply nobody will read."""
|
||||
admin = _admin_contact()
|
||||
body = (
|
||||
"Hi,\n\n"
|
||||
"Your request to use Siftlode has been approved — welcome aboard.\n\n"
|
||||
"Open Siftlode and sign in, and your YouTube subscriptions feed will be ready:\n\n"
|
||||
f"{settings.app_base}\n\n"
|
||||
"If you weren't expecting this, you can ignore the message.\n\n"
|
||||
"— Siftlode" + (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
|
||||
return admin, (mailtext.text("contact.reply", lang, admin=admin) if admin else "")
|
||||
|
||||
|
||||
def send_access_approved(
|
||||
email: str,
|
||||
lang: str,
|
||||
verify_link: str | None = None,
|
||||
has_account: bool = False,
|
||||
hours: int = 24,
|
||||
) -> bool:
|
||||
"""Answer someone who ASKED for access. Sent automatically on approval, because a person who
|
||||
made a request is owed a reply — unlike the admin-initiated invitation below."""
|
||||
admin, contact = _reply_contact(lang)
|
||||
body = mailtext.text(
|
||||
"access_approved.body",
|
||||
lang,
|
||||
next_step=_next_step(verify_link, has_account, hours, lang),
|
||||
contact=contact,
|
||||
)
|
||||
return _send([email], "You're in — your Siftlode access is approved", body, reply_to=admin)
|
||||
return _send([email], mailtext.text("access_approved.subject", lang), body, reply_to=admin)
|
||||
|
||||
|
||||
def send_access_denied(email: str) -> bool:
|
||||
admin = _admin_contact()
|
||||
body = (
|
||||
"Hi,\n\n"
|
||||
"Thanks for your interest in Siftlode. After review, your access request wasn't approved "
|
||||
"this time.\n\n"
|
||||
"If you think this was a mistake, you're welcome to reply and let us know.\n\n"
|
||||
"— Siftlode" + (f"\n\nQuestions? Just reply to this email ({admin})." if admin else "")
|
||||
def send_access_invitation(
|
||||
email: str,
|
||||
lang: str,
|
||||
verify_link: str | None = None,
|
||||
has_account: bool = False,
|
||||
hours: int = 24,
|
||||
) -> bool:
|
||||
"""Invite someone the admin let in WITHOUT them asking — sent only when the admin presses send.
|
||||
|
||||
Whitelisting an address is opening a door, not knocking on one: the operator has usually already
|
||||
agreed it with the person, and an address can sit approved for weeks without anyone registering.
|
||||
So adding to the whitelist stays silent and this is a separate, deliberate act. That is also why
|
||||
the closing line offers no deadline — nudging is the one thing this mail must not do.
|
||||
|
||||
Its language is the one place the app has no signal to read: the admin typed this address in, so
|
||||
nobody has ever told us how to address its owner. That is why the send button asks (E9.5 S7).
|
||||
"""
|
||||
admin, contact = _reply_contact(lang)
|
||||
body = mailtext.text(
|
||||
"access_invitation.body",
|
||||
lang,
|
||||
next_step=_next_step(verify_link, has_account, hours, lang),
|
||||
contact=contact,
|
||||
)
|
||||
return _send([email], "About your Siftlode access request", body, reply_to=admin)
|
||||
return _send([email], mailtext.text("access_invitation.subject", lang), body, reply_to=admin)
|
||||
|
||||
|
||||
def send_role_changed(to: str, role: str, operator: str | None) -> bool:
|
||||
line = (
|
||||
"You've been granted admin access on Siftlode — you can now manage settings, the "
|
||||
"scheduler and users."
|
||||
if role == "admin"
|
||||
else "Your admin access on Siftlode has been removed — you're now a regular user."
|
||||
def send_access_denied(email: str, lang: str) -> bool:
|
||||
admin, contact = _reply_contact(lang)
|
||||
body = mailtext.text("access_denied.body", lang, contact=contact)
|
||||
return _send([email], mailtext.text("access_denied.subject", lang), body, reply_to=admin)
|
||||
|
||||
|
||||
def send_role_changed(to: str, lang: str, role: str, operator: str | None) -> bool:
|
||||
line = mailtext.text(
|
||||
"role_changed.granted" if role == "admin" else "role_changed.revoked", lang
|
||||
)
|
||||
contact = f"\n\nQuestions? Contact the operator at {operator}." if operator else ""
|
||||
body = "Hi,\n\n" + line + contact + "\n\n— Siftlode"
|
||||
return _send([to], "Your Siftlode role has changed", body, reply_to=operator)
|
||||
contact = mailtext.text("contact.operator", lang, operator=operator) if operator else ""
|
||||
body = mailtext.text("role_changed.body", lang, line=line, contact=contact)
|
||||
return _send([to], mailtext.text("role_changed.subject", lang), body, reply_to=operator)
|
||||
|
||||
|
||||
def send_account_unsuspended(to: str, operator: str | None) -> bool:
|
||||
contact = f"\n\nQuestions? Contact the operator at {operator}." if operator else ""
|
||||
body = (
|
||||
"Hi,\n\n"
|
||||
"Your Siftlode account has been reinstated — the suspension was lifted and you can sign "
|
||||
"in again." + contact + "\n\n— Siftlode"
|
||||
)
|
||||
return _send([to], "Your Siftlode account has been reinstated", body, reply_to=operator)
|
||||
def send_account_unsuspended(to: str, lang: str, operator: str | None) -> bool:
|
||||
contact = mailtext.text("contact.operator", lang, operator=operator) if operator else ""
|
||||
body = mailtext.text("account_unsuspended.body", lang, contact=contact)
|
||||
return _send([to], mailtext.text("account_unsuspended.subject", lang), body, reply_to=operator)
|
||||
|
||||
|
||||
def send_account_deleted(to: str, operator: str | None) -> bool:
|
||||
def send_account_deleted(to: str, lang: str, operator: str | None) -> bool:
|
||||
contact = mailtext.text("contact.deleted", lang, operator=operator) if operator else ""
|
||||
body = mailtext.text("account_deleted.body", lang, contact=contact)
|
||||
return _send([to], mailtext.text("account_deleted.subject", lang), body, reply_to=operator)
|
||||
|
||||
|
||||
def send_account_suspended(to: str, lang: str, operator: str | None) -> bool:
|
||||
contact = (
|
||||
f"\n\nIf you didn't request this, contact the Siftlode operator at {operator}."
|
||||
mailtext.text("contact.suspended", lang, operator=operator)
|
||||
if operator
|
||||
else ""
|
||||
else mailtext.text("contact.suspended_unknown", lang)
|
||||
)
|
||||
body = (
|
||||
"Hi,\n\n"
|
||||
"Your Siftlode account and all associated data — subscriptions, tags, watch history, "
|
||||
"playlists and settings — have been permanently deleted. Nothing is retained.\n\n"
|
||||
"Thanks for having used Siftlode." + contact + "\n\n— Siftlode"
|
||||
body = mailtext.text("account_suspended.body", lang, contact=contact)
|
||||
return _send([to], mailtext.text("account_suspended.subject", lang), body, reply_to=operator)
|
||||
|
||||
|
||||
def send_admin_new_request(admins: list[str], lang: str, requester: str) -> bool:
|
||||
"""Tell the admins someone is waiting. One call per language group (see
|
||||
`routes.auth.notify_admins_of_request`) — admins are people with their own UI language, and a
|
||||
single blast to all of them would have to pick one and be wrong for the rest.
|
||||
|
||||
The link deep-links straight to the admin Access-requests page (the SPA reads ?admin=…). The
|
||||
requester address is spelled out (mail clients auto-link both the URL and the address), and the
|
||||
reply note is accurate: _send sets Reply-To to the requester, so a reply reaches THEM — not the
|
||||
sending mailbox. (The old copy said "Settings → Account", which was the wrong menu.)
|
||||
"""
|
||||
body = mailtext.text(
|
||||
"admin_new_request.body",
|
||||
lang,
|
||||
requester=requester,
|
||||
link=f"{settings.app_base}/?admin=access-requests",
|
||||
)
|
||||
return _send([to], "Your Siftlode account has been deleted", body, reply_to=operator)
|
||||
subject = mailtext.text("admin_new_request.subject", lang, requester=requester)
|
||||
return _send(admins, subject, body, reply_to=requester)
|
||||
|
||||
|
||||
def send_account_suspended(to: str, operator: str | None) -> bool:
|
||||
contact = (
|
||||
f"If you think this is a mistake or have questions, contact the Siftlode operator "
|
||||
f"at {operator}.\n\n"
|
||||
if operator
|
||||
else "If you think this is a mistake, please contact the Siftlode operator.\n\n"
|
||||
def send_verify_email(to: str, lang: str, link: str, hours: int = 24) -> bool:
|
||||
# The body says what the link's rules are. The reset mail always has; this one left the reader to
|
||||
# discover both by meeting a dead link, which is precisely the silence this epic removes.
|
||||
body = mailtext.text(
|
||||
"verify_email.body", lang, link=link, expiry=mailtext.link_expiry(hours, lang)
|
||||
)
|
||||
body = (
|
||||
"Hi,\n\n"
|
||||
"A sign-in to your Siftlode account was just attempted, but the account is currently "
|
||||
"suspended, so access was denied.\n\n" + contact + "— Siftlode"
|
||||
)
|
||||
return _send([to], "Your Siftlode account is suspended", body, reply_to=operator)
|
||||
return _send([to], mailtext.text("verify_email.subject", lang), body)
|
||||
|
||||
|
||||
def send_admin_new_request(admins: list[str], requester: str) -> bool:
|
||||
# The link deep-links straight to the admin Access-requests page (the SPA reads ?admin=…). The
|
||||
# requester address is spelled out (mail clients auto-link both the URL and the address), and the
|
||||
# reply note is accurate: _send sets Reply-To to the requester, so a reply reaches THEM — not the
|
||||
# sending mailbox. (The old copy said "Settings → Account", which was the wrong menu.)
|
||||
approve_url = f"{settings.app_base}/?admin=access-requests"
|
||||
body = (
|
||||
f"{requester} just requested access to Siftlode.\n\n"
|
||||
f"Approve or deny it here:\n{approve_url}\n"
|
||||
f"(or in Siftlode: open Users → Access requests)\n\n"
|
||||
f"Requester's email: {requester}\n"
|
||||
f"Replying to this message goes straight to the requester (it's the Reply-To address).\n"
|
||||
)
|
||||
return _send(admins, f"Siftlode access request from {requester}", body, reply_to=requester)
|
||||
def send_password_reset(to: str, lang: str, link: str) -> bool:
|
||||
body = mailtext.text("password_reset.body", lang, link=link)
|
||||
return _send([to], mailtext.text("password_reset.subject", lang), body)
|
||||
|
||||
|
||||
def send_verify_email(to: str, link: str) -> bool:
|
||||
body = (
|
||||
"Hi,\n\n"
|
||||
"Confirm your email to finish creating your Siftlode account:\n\n"
|
||||
f"{link}\n\n"
|
||||
"If you didn't sign up, you can ignore this message.\n\n"
|
||||
"— Siftlode"
|
||||
)
|
||||
return _send([to], "Confirm your Siftlode email", body)
|
||||
|
||||
|
||||
def send_password_reset(to: str, link: str) -> bool:
|
||||
body = (
|
||||
"Hi,\n\n"
|
||||
"Use this link to set a new Siftlode password (it expires in an hour):\n\n"
|
||||
f"{link}\n\n"
|
||||
"If you didn't request this, you can ignore this message — your password is unchanged.\n\n"
|
||||
"— Siftlode"
|
||||
)
|
||||
return _send([to], "Reset your Siftlode password", body)
|
||||
|
||||
|
||||
def send_test(to: str) -> bool:
|
||||
body = (
|
||||
"This is a test email from Siftlode.\n\n"
|
||||
"If you're reading this, your SMTP settings work.\n\n"
|
||||
"— Siftlode"
|
||||
)
|
||||
return _send([to], "Siftlode SMTP test", body)
|
||||
def send_test(to: str, lang: str) -> bool:
|
||||
return _send([to], mailtext.text("test.subject", lang), mailtext.text("test.body", lang))
|
||||
|
||||
+66
-14
@@ -30,7 +30,7 @@ from app.models import (
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.utils import iso, to_tsquery_str
|
||||
from app.utils import iso, like_escape, to_tsquery_str
|
||||
|
||||
|
||||
def _channel_url(channel_id: str, handle: str | None) -> str:
|
||||
@@ -100,6 +100,11 @@ def serialize(row) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _tag_visible(user: User):
|
||||
"""A ChannelTag row this user may see: a shared system tag (user_id NULL) or their own."""
|
||||
return or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||
|
||||
|
||||
def tags_by_category(db: Session, tag_ids: list[int]) -> dict[str, list[int]]:
|
||||
"""Group tag ids by their category (language/topic/other). The single source for the category split
|
||||
shared by the feed's tag WHERE (filtered_query) and the facet/count summary (feed_summary), so the
|
||||
@@ -112,12 +117,26 @@ def tags_by_category(db: Session, tag_ids: list[int]) -> dict[str, list[int]]:
|
||||
return out
|
||||
|
||||
|
||||
def not_blocked_cond(user: User):
|
||||
"""Per-user channel blocklist: never show this user videos from a channel they blocked
|
||||
(applies to feed, Library, explore, the channel page and the Home resume rail alike).
|
||||
|
||||
A shared expression rather than a clause each surface writes for itself. Blocking does not touch
|
||||
`video_states`, so a half-watched video from a newly blocked channel survives with its resume
|
||||
position — and the one query that copied only PART of this predicate set is exactly how that
|
||||
video came back as the landing page's suggestion after the user had said "never again"."""
|
||||
return Video.channel_id.notin_(
|
||||
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
|
||||
)
|
||||
|
||||
|
||||
def filtered_query(
|
||||
db: Session,
|
||||
user: User,
|
||||
*,
|
||||
tags: list[int],
|
||||
tag_modes: dict[str, str],
|
||||
untagged: bool = False,
|
||||
channel_ids: list[str],
|
||||
q: str | None,
|
||||
min_duration: int | None,
|
||||
@@ -201,13 +220,7 @@ def filtered_query(
|
||||
)
|
||||
)
|
||||
|
||||
# Per-user channel blocklist: never show this user videos from a channel they blocked
|
||||
# (applies to feed, Library, explore, and the channel page alike).
|
||||
query = query.where(
|
||||
Video.channel_id.notin_(
|
||||
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
|
||||
)
|
||||
)
|
||||
query = query.where(not_blocked_cond(user))
|
||||
|
||||
# Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
|
||||
# (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
|
||||
@@ -259,8 +272,9 @@ def filtered_query(
|
||||
# `unaccent(channels.title) ILIKE '%q%'` straight into the per-video predicate defeated the
|
||||
# search_vector GIN — the planner detoasted every non-matching tsvector to test the `@@`
|
||||
# (~2.6s/search on the prod-sized set); channel_id.in_(...) keeps the GIN as a BitmapOr branch
|
||||
# (PERF-12). Empty match -> false() so the OR collapses to the text match alone.
|
||||
like = func.unaccent(f"%{q}%")
|
||||
# (PERF-12). Empty match -> false() so the OR collapses to the text match alone. The query
|
||||
# is a literal inside the pattern: `_` in `my_channel` is not a wildcard.
|
||||
like = func.unaccent(f"%{like_escape(q)}%")
|
||||
matching_channel_ids = (
|
||||
db.execute(select(Channel.id).where(func.unaccent(Channel.title).ilike(like)))
|
||||
.scalars()
|
||||
@@ -294,7 +308,7 @@ def filtered_query(
|
||||
# selections but still honours the other categories' filters).
|
||||
if exclude_tag_category:
|
||||
by_category.pop(exclude_tag_category, None)
|
||||
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||
visible = _tag_visible(user)
|
||||
for category, ids in by_category.items():
|
||||
if tag_modes.get(category, "or") == "and" and len(ids) > 1:
|
||||
sub = (
|
||||
@@ -307,6 +321,18 @@ def filtered_query(
|
||||
sub = select(ChannelTag.channel_id).where(ChannelTag.tag_id.in_(ids), visible)
|
||||
query = query.where(Video.channel_id.in_(sub))
|
||||
|
||||
# "No tag" filter (item B): keep only channels the USER has not personally tagged in the channel
|
||||
# manager — i.e. channels with no ChannelTag row owned by this user (user_id == me, source "user").
|
||||
# System/auto tags (user_id NULL) don't count: this is "channels I haven't tagged yet". It's a plain
|
||||
# channel-level narrowing (like channel_ids/date), NOT a tag-category selection, so feed_summary
|
||||
# leaves it applied in the base pass and needs no mirror in passes().
|
||||
if untagged:
|
||||
query = query.where(
|
||||
Video.channel_id.notin_(
|
||||
select(ChannelTag.channel_id).where(ChannelTag.user_id == user.id)
|
||||
)
|
||||
)
|
||||
|
||||
# Content type: Normal / Shorts / Live·Upcoming as a union of enabled types.
|
||||
# Explicit Watched/Saved/Hidden views show every type so nothing goes missing.
|
||||
explicit_view = show in ("watched", "hidden")
|
||||
@@ -355,8 +381,9 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
||||
within, a category in "and" mode requires ALL its selected tags)."""
|
||||
tag_modes = params.get("tag_modes", {})
|
||||
|
||||
# ONE pass: per-channel video counts for the base filter, tags stripped so every candidate channel
|
||||
# is present to facet over.
|
||||
# ONE pass: per-channel video counts for the base filter, the positive tag filter stripped so every
|
||||
# candidate channel is present to facet over. The "No tag" (untagged) filter is a plain channel-level
|
||||
# narrowing, so it STAYS applied here (not stripped) and the facets naturally drill down within it.
|
||||
base, _rank = filtered_query(db, user, **{**params, "tags": []})
|
||||
ch_col = base.with_only_columns(Video.channel_id).subquery()
|
||||
channel_counts: dict[str, int] = dict(
|
||||
@@ -386,7 +413,8 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
||||
|
||||
def passes(ch_tags: set[int], exclude_cat: str | None) -> bool:
|
||||
# The Python mirror of filtered_query's SQL tag block (AND across categories, OR within,
|
||||
# topic-AND requires all selected) — keep in lockstep; see the ⚠️ note there.
|
||||
# topic-AND requires all selected) — keep in lockstep; see the ⚠️ note there. (The "No tag"
|
||||
# filter is applied in the base query above, so it needs no mirror here.)
|
||||
for cat, sel in selected_by_cat.items():
|
||||
if cat == exclude_cat:
|
||||
continue
|
||||
@@ -417,6 +445,30 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
||||
return {"count": count, "facets": facets}
|
||||
|
||||
|
||||
def feed_years(db: Session, user: User, **params) -> list[dict]:
|
||||
"""Per-year video counts for the current filter context (item 8 / E8·S2): powers the year control's
|
||||
quick chips + the "more…" per-year-counts dialog. Every OTHER filter (tags, channels, content type,
|
||||
search, watch state, scope) still applies, but the publish-date narrowing itself is DROPPED —
|
||||
published_after/before and max_age_days — so the year list is the full set of years available and
|
||||
switching years stays a live drill-down (the same principle as the tag facets dropping their own
|
||||
category). One grouped query, newest year first."""
|
||||
base, _rank = filtered_query(
|
||||
db,
|
||||
user,
|
||||
**{**params, "published_after": None, "published_before": None, "max_age_days": None},
|
||||
)
|
||||
pub = base.with_only_columns(Video.published_at.label("pub")).subquery()
|
||||
# Bucket by UTC year to stay in lockstep with the year filter's boundaries (published_after/before
|
||||
# are pinned to UTC in filtered_query). published_at is timestamptz, so a bare extract('year', …)
|
||||
# would use the DB session timezone and, on a non-UTC session, bucket a video near a year boundary
|
||||
# into a different year than the filter includes — the count and the drill-down feed would disagree.
|
||||
year = func.extract("year", func.timezone("UTC", pub.c.pub))
|
||||
rows = db.execute(
|
||||
select(year.label("y"), func.count()).group_by(year).order_by(year.desc())
|
||||
).all()
|
||||
return [{"year": int(y), "count": n} for y, n in rows if y is not None]
|
||||
|
||||
|
||||
# --- Keyset (cursor) pagination ---------------------------------------------
|
||||
#
|
||||
# Each sort is described as an ordered list of key columns (expression + direction
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Bug reports built from debug sessions (E16): the files an issue carries, and where they go —
|
||||
downloaded as a zip, or (S4) sent to a Gitea issue tracker. The session itself — what is recorded
|
||||
and tagged — is the core module `app/debugsession.py`."""
|
||||
@@ -0,0 +1,79 @@
|
||||
"""What an issue from a debug session carries, as files — one definition for the zip export and the
|
||||
Gitea upload, so the two cannot drift apart.
|
||||
|
||||
* `issue.md` — the title and the body the user wrote and reviewed in the composer (rendered there,
|
||||
in their language; the server does not rewrite it);
|
||||
* `client.log` / `server.log` — the session's evidence (`debugsession.bundle`) minus the rows the
|
||||
user took out, one JSON object per line, browser rows and server rows apart. `.log`, not
|
||||
`.jsonl`: Gitea refuses that extension (measured, plans/e16 S0);
|
||||
* the screenshots, in the order they were taken.
|
||||
|
||||
Every row is already redacted twice (at capture, and again when stored), and nothing here adds a
|
||||
field the review list did not show.
|
||||
"""
|
||||
|
||||
import io
|
||||
import json
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, logsink
|
||||
from app.models import DebugScreenshot, DebugSession
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IssueFile:
|
||||
name: str
|
||||
data: bytes
|
||||
mime: str
|
||||
|
||||
|
||||
def title_of(session: DebugSession) -> str:
|
||||
# Not a sentence of ours: the composer always saves a title in the user's language, and a
|
||||
# session exported before it ever saved is named by its date alone, in no language at all.
|
||||
title = ((session.draft or {}).get("title") or "").strip()
|
||||
return title or f"{session.started_at:%Y-%m-%d %H:%M} · {session.id}"
|
||||
|
||||
|
||||
def body_of(session: DebugSession) -> str:
|
||||
return ((session.draft or {}).get("body") or "").strip()
|
||||
|
||||
|
||||
def _lines(rows) -> bytes:
|
||||
return "".join(
|
||||
json.dumps(logsink.serialize(r), ensure_ascii=False, default=str) + "\n" for r in rows
|
||||
).encode()
|
||||
|
||||
|
||||
def files(db: Session, session: DebugSession) -> list[IssueFile]:
|
||||
"""The session's issue as files: the text, the two logs, the screenshots."""
|
||||
rows, _ = debugsession.bundle(db, session, drop_excluded=True)
|
||||
text = f"# {title_of(session)}\n\n{body_of(session)}\n"
|
||||
out = [
|
||||
IssueFile("issue.md", text.encode(), "text/markdown"),
|
||||
IssueFile("client.log", _lines(r for r in rows if r.source == "web"), "text/plain"),
|
||||
IssueFile("server.log", _lines(r for r in rows if r.source != "web"), "text/plain"),
|
||||
]
|
||||
shots = db.scalars(
|
||||
select(DebugScreenshot)
|
||||
.where(DebugScreenshot.session_id == session.id)
|
||||
.order_by(DebugScreenshot.id)
|
||||
)
|
||||
for n, shot in enumerate(shots, start=1):
|
||||
ext = debugsession.IMAGE_TYPES[shot.mime]
|
||||
name = f"{debugsession.SCREENSHOT_PREFIX}{n}.{ext}"
|
||||
out.append(IssueFile(name, shot.data, shot.mime))
|
||||
return out
|
||||
|
||||
|
||||
def zipped(items: list[IssueFile]) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
|
||||
for item in items:
|
||||
# The images are compressed already: deflating them again costs time and wins nothing.
|
||||
method = zipfile.ZIP_STORED if item.mime.startswith("image/") else zipfile.ZIP_DEFLATED
|
||||
z.writestr(item.name, item.data, compress_type=method)
|
||||
return buf.getvalue()
|
||||
@@ -0,0 +1,275 @@
|
||||
"""The Gitea issue tracker a debug session's issue goes to (E16 S4) — measured against Gitea 1.27.1
|
||||
before a line of this was written (plans/e16-debug-issue.md, S0).
|
||||
|
||||
Configured by an admin (sysconfig `issues_gitea_url` / `_repo` / `_token`, the token encrypted at
|
||||
rest). The repository must be PRIVATE: an issue carries its reporter's log — page titles, filter
|
||||
values, at Teljes what they typed into search — and a public repository would publish it. That is
|
||||
checked on every send, not only in the admin's "test connection", because a repository can be made
|
||||
public after the fact.
|
||||
|
||||
What an issue is made of comes from `bundle.files` (the same files the zip export carries). Sending
|
||||
is three kinds of call, measured:
|
||||
* `POST /repos/{repo}/issues` — the title and the body the user wrote;
|
||||
* `POST /repos/{repo}/issues/{n}/assets?name=…` — ONE file per request (field `attachment`);
|
||||
* `PATCH /repos/{repo}/issues/{n}` — the body again, with the screenshots shown inline once their
|
||||
links exist.
|
||||
A private repository's attachment answers 404 to a browser not signed in to Gitea, which is why the
|
||||
composer's own preview shows our copies, never these links.
|
||||
|
||||
Delivery keeps its progress in the session's draft (`delivery`: the issue number, the files still to
|
||||
upload, the links of those uploaded, whether the body shows the screenshots yet), so a failure
|
||||
half-way keeps the issue that exists and a retry does only what is missing — never a second issue.
|
||||
Every step is checkpointed (the caller's `db.commit`), so what did happen is kept whatever fails
|
||||
next — and the first checkpoint comes before the first call, with the tracker's settings read ABOVE
|
||||
it, so neither a row lock nor a read snapshot nor a pooled connection is held across the calls. The
|
||||
caller claims the session for the send first (`debugsession.claim_send`) and passes the claim's
|
||||
token in, which is what keeps a second send, an autosave or a screenshot out of the way; the claim
|
||||
is renewed by every progress report, and a report whose claim is gone raises rather than writing
|
||||
over the send that took it. `debugsession.not_uploaded` / `images_pending` read that progress.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, logs, sysconfig
|
||||
from app.issues import bundle
|
||||
from app.models import DebugSession
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
TIMEOUT_SECONDS = 20.0
|
||||
|
||||
Reason = Literal[
|
||||
"not_configured", # url, repository or token missing
|
||||
"unreachable", # no answer, or a 5xx
|
||||
"bad_token", # 401/403: the token is wrong or lacks write:issue / read:repository
|
||||
"repo_not_found", # 404 on the repository
|
||||
"public_repo", # the repository is public — an issue would publish the reporter's log
|
||||
"issues_disabled", # the repository has its issue tracker switched off
|
||||
"rejected", # any other 4xx (a file type the instance does not allow, …), or an answer that is
|
||||
# not the API's (a redirect — an http:// address behind https —, a page that is not JSON)
|
||||
]
|
||||
|
||||
|
||||
class GiteaError(Exception):
|
||||
def __init__(self, reason: Reason) -> None:
|
||||
super().__init__(reason)
|
||||
self.reason: Reason = reason
|
||||
|
||||
|
||||
def _settings(db: Session) -> tuple[str, str, str]:
|
||||
return (
|
||||
sysconfig.get_str(db, "issues_gitea_url").strip().rstrip("/"),
|
||||
sysconfig.get_str(db, "issues_gitea_repo").strip().strip("/"),
|
||||
sysconfig.get_str(db, "issues_gitea_token").strip(),
|
||||
)
|
||||
|
||||
|
||||
def target(db: Session) -> dict | None:
|
||||
"""Where a report goes, for the composer to say so — the repository — or None when no tracker is
|
||||
configured (the report can then only be downloaded)."""
|
||||
url, repo, token = _settings(db)
|
||||
return {"repo": repo} if url and repo and token else None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _client(settings: tuple[str, str, str]) -> Iterator[tuple[httpx.Client, str]]:
|
||||
# Takes the settings already read, not the session: reading them HERE opened a transaction one
|
||||
# line after `deliver` had ended one, and held it across the calls below (deep-review round 4).
|
||||
url, repo, token = settings
|
||||
if not (url and repo and token):
|
||||
raise GiteaError("not_configured")
|
||||
with httpx.Client(
|
||||
base_url=f"{url}/api/v1",
|
||||
headers={"Authorization": f"token {token}", "Accept": "application/json"},
|
||||
timeout=TIMEOUT_SECONDS,
|
||||
) as client:
|
||||
yield client, repo
|
||||
|
||||
|
||||
def _call(client: httpx.Client, method: str, path: str, **kwargs) -> httpx.Response:
|
||||
try:
|
||||
r = client.request(method, path, **kwargs)
|
||||
except httpx.HTTPError as exc:
|
||||
log.warning(
|
||||
"the issue tracker did not answer",
|
||||
extra=logs.event("issues.gitea.unreachable", path=path, error=type(exc).__name__),
|
||||
)
|
||||
raise GiteaError("unreachable") from None
|
||||
if r.status_code < 300:
|
||||
return r
|
||||
# The status only: Gitea's message can quote back what was sent — the user's own title.
|
||||
log.warning(
|
||||
"the issue tracker refused a call",
|
||||
extra=logs.event("issues.gitea.refused", path=path, status=r.status_code),
|
||||
)
|
||||
if r.status_code in (401, 403):
|
||||
raise GiteaError("bad_token")
|
||||
if r.status_code == 404:
|
||||
raise GiteaError("repo_not_found")
|
||||
if r.status_code >= 500:
|
||||
raise GiteaError("unreachable")
|
||||
raise GiteaError("rejected")
|
||||
|
||||
|
||||
def _json(r: httpx.Response, *keys: str) -> dict[str, Any]:
|
||||
"""The answer as the API's JSON, holding `keys` — anything else is refused, not a crash, so a
|
||||
step after the issue was created cannot take down the request that holds its record.
|
||||
|
||||
The CREATE call is the one place this cannot keep that record: its answer IS the record. An
|
||||
answer it cannot read there leaves the session unsent, and a retry opens another issue — the
|
||||
price of not being able to tell a refused create from an unreadable one."""
|
||||
try:
|
||||
data = r.json()
|
||||
# silent-ok: not JSON is one of the answers refused (and logged) just below.
|
||||
except ValueError:
|
||||
data = None
|
||||
if not isinstance(data, dict) or any(k not in data for k in keys):
|
||||
log.warning(
|
||||
"the issue tracker answered something that is not its API",
|
||||
extra=logs.event("issues.gitea.unexpected", status=r.status_code),
|
||||
)
|
||||
raise GiteaError("rejected")
|
||||
return data
|
||||
|
||||
|
||||
def _private_repo(client: httpx.Client, repo: str) -> dict:
|
||||
data = _json(_call(client, "GET", f"/repos/{repo}"))
|
||||
if not data.get("private"):
|
||||
raise GiteaError("public_repo")
|
||||
if not data.get("has_issues"):
|
||||
raise GiteaError("issues_disabled")
|
||||
return data
|
||||
|
||||
|
||||
def check(db: Session) -> dict:
|
||||
"""The admin's "test connection": the repository answers, is private and takes issues."""
|
||||
with _client(_settings(db)) as (client, repo):
|
||||
data = _private_repo(client, repo)
|
||||
return {"repo": data.get("full_name") or repo, "url": data.get("html_url")}
|
||||
|
||||
|
||||
def preview(db: Session, text: str) -> str:
|
||||
"""The body as Gitea itself will render it (its markdown flavour, its sanitiser)."""
|
||||
with _client(_settings(db)) as (client, repo):
|
||||
r = _call(client, "POST", "/markdown", json={"Text": text, "Mode": "gfm", "Context": repo})
|
||||
return r.text
|
||||
|
||||
|
||||
def deliver(
|
||||
db: Session,
|
||||
session: DebugSession,
|
||||
*,
|
||||
token: str,
|
||||
checkpoint: Callable[[], None],
|
||||
before_create: Callable[[], None] | None = None,
|
||||
on_created: Callable[[], None] | None = None,
|
||||
) -> str:
|
||||
"""Send the session's issue, or finish sending it; returns its URL.
|
||||
|
||||
`token` is the caller's claim on this session (`debugsession.claim_send`). Every progress report
|
||||
renews it and checks the row still holds it, so a send that outlived its lease stops here rather
|
||||
than writing over the one that took over (`debugsession.SendClaimLost`).
|
||||
|
||||
`checkpoint` persists what has happened so far and is called after every step (the caller's
|
||||
commit — this module does not decide when a transaction ends). `before_create` runs right before
|
||||
the tracker is asked to open a NEW issue, after the checks that can refuse it without asking,
|
||||
and may raise to stop it; `on_created` runs right after one was opened, in time to be committed
|
||||
by the SAME checkpoint that first writes `issue_url` — anything the caller must not be able to
|
||||
lose separately from the link belongs there.
|
||||
"""
|
||||
state = debugsession.delivery(session)
|
||||
files = bundle.files(db, session)
|
||||
attachments = [f for f in files if f.name != "issue.md"]
|
||||
settings = _settings(db)
|
||||
# Reading the bundle and the tracker's settings opened a transaction; end it before the first
|
||||
# call, so nothing holds a snapshot (and an idle connection) across minutes of tracker I/O.
|
||||
# The settings are read HERE and handed to `_client`, not read inside it: reading them there
|
||||
# was a SELECT one line after this commit, which reopened the transaction and held it — idle,
|
||||
# with its snapshot pinned — across the repository check and the create (deep-review round 4
|
||||
# measured it in pg_stat_activity). A comment saying otherwise sat right here.
|
||||
checkpoint()
|
||||
with _client(settings) as (client, repo):
|
||||
_private_repo(client, repo)
|
||||
if "number" not in state:
|
||||
if before_create is not None:
|
||||
before_create()
|
||||
created = _json(
|
||||
_call(
|
||||
client,
|
||||
"POST",
|
||||
f"/repos/{repo}/issues",
|
||||
json={"title": bundle.title_of(session), "body": bundle.body_of(session)},
|
||||
),
|
||||
"number",
|
||||
"html_url",
|
||||
)
|
||||
# Built ON the state read at the start so nothing already in it is dropped. The claim
|
||||
# is NOT among those things, whatever this comment used to say: `set_delivery` strips
|
||||
# any `sending` passed to it and stamps the row's own from `token` (round 4 — removing
|
||||
# the spread left the whole suite green, because the claim never travelled here).
|
||||
state = {
|
||||
**state,
|
||||
"number": created["number"],
|
||||
"pending": [f.name for f in attachments],
|
||||
"links": {},
|
||||
}
|
||||
session.issue_url = created["html_url"]
|
||||
try:
|
||||
debugsession.set_delivery(db, session, state, token)
|
||||
except debugsession.SendClaimLost:
|
||||
# The one step where losing the claim costs something irreversible: the issue is
|
||||
# already open on the tracker and this is what would have recorded its link. We
|
||||
# still must not write — another send owns the row and is opening its own issue —
|
||||
# so say which issue was left behind, because nothing else will.
|
||||
# noqa TRY400: the traceback is one SendClaimLost carrying the session id and
|
||||
# nothing else; the issue number below is the whole news.
|
||||
log.error( # noqa: TRY400
|
||||
"an issue was created by a send that had already lost its claim",
|
||||
extra=logs.event(
|
||||
"issues.gitea.orphaned",
|
||||
number=created["number"],
|
||||
session_id=session.id,
|
||||
),
|
||||
)
|
||||
raise
|
||||
if on_created is not None:
|
||||
on_created()
|
||||
checkpoint()
|
||||
log.info(
|
||||
"issue created",
|
||||
extra=logs.event("issues.gitea.created", number=created["number"]),
|
||||
)
|
||||
number = state["number"]
|
||||
for item in attachments:
|
||||
if item.name not in state["pending"]:
|
||||
continue
|
||||
asset = _json(
|
||||
_call(
|
||||
client,
|
||||
"POST",
|
||||
f"/repos/{repo}/issues/{number}/assets",
|
||||
params={"name": item.name},
|
||||
files={"attachment": (item.name, item.data, item.mime)},
|
||||
),
|
||||
"browser_download_url",
|
||||
)
|
||||
state = {
|
||||
**state,
|
||||
"pending": [n for n in state["pending"] if n != item.name],
|
||||
"links": {**state["links"], item.name: asset["browser_download_url"]},
|
||||
}
|
||||
debugsession.set_delivery(db, session, state, token)
|
||||
checkpoint()
|
||||
shots = [n for n in state["links"] if n.startswith(debugsession.SCREENSHOT_PREFIX)]
|
||||
if shots and not state.get("images_shown"):
|
||||
images = "\n".join(f"" for n in sorted(shots))
|
||||
body = f"{bundle.body_of(session)}\n\n---\n\n{images}"
|
||||
_call(client, "PATCH", f"/repos/{repo}/issues/{number}", json={"body": body})
|
||||
debugsession.set_delivery(db, session, {**state, "images_shown": True}, token)
|
||||
checkpoint()
|
||||
return session.issue_url or ""
|
||||
@@ -0,0 +1,839 @@
|
||||
"""Structured logging for every Siftlode process — the ONE place logging is configured.
|
||||
|
||||
Before this module a single deployment printed FOUR different log formats: uvicorn's
|
||||
`log_config.json` for the API, a hand-rolled StreamHandler in `main.py` for everything that did not
|
||||
start under uvicorn (tests, scripts), `logging.basicConfig()` in the worker and in the download
|
||||
child (no timestamp at all), and alembic's `fileConfig`. None of them carried a correlation id, and
|
||||
uvicorn's access log printed the query string of every request (`?account=1`, `?source=feed`) — so
|
||||
the logs were both unjoinable and leaky. `configure_logging()` replaces all four.
|
||||
|
||||
Event shape — the OpenTelemetry log data model, without the SDK; one JSON object per line:
|
||||
ts · level · sev · source · logger · event · msg · attrs
|
||||
plus whatever correlation the context carries (`request_id`, `user_id`, `debug_session_id`,
|
||||
`job_id`) and the app version. That shape is what E22.6's admin console filters on and what a Gitea
|
||||
issue attachment will carry, so it is deliberately flat and stable.
|
||||
|
||||
Call sites stay ordinary stdlib logging:
|
||||
|
||||
log = logs.get_logger(__name__) # app/downloads/pipeline.py -> siftlode.downloads.pipeline
|
||||
log.info("job finished", extra=logs.event("download.job.done", job_id=7, bytes=n))
|
||||
|
||||
`event()` is optional — a bare `log.info("…")` still comes out well-formed — but the dotted code is
|
||||
what makes a line queryable later, so add one wherever the line matters.
|
||||
|
||||
REDACTION IS APPLIED AT THE HANDLER, not at the call site: every message, every attr value and
|
||||
every traceback passes `redact()` on the way out. That is the whole point — a secret cannot be
|
||||
leaked by forgetting to redact it at one of ~125 call sites, and a new call site is safe by
|
||||
default. Never logged, at any level: passwords, tokens (session, reset/verify, setup, share,
|
||||
OAuth), cookie blobs, secret config values, and email addresses.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
import json
|
||||
import logging
|
||||
import logging.config
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from app.config import settings
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
|
||||
# One level below DEBUG for the "Teljes / TRACE" diagnostics level (search strings, redacted request
|
||||
# bodies). Registered on import so `logging.getLogger(...).log(logs.TRACE, …)` and a `LOG_LEVEL=TRACE`
|
||||
# both work in every process, including one that never calls configure_logging (a test).
|
||||
TRACE = 5
|
||||
logging.addLevelName(TRACE, "TRACE")
|
||||
|
||||
# OpenTelemetry severity numbers for our level names. Kept beside the level because the number is
|
||||
# what a log backend sorts and filters on; the name is for humans reading a terminal.
|
||||
SEVERITY: dict[str, int] = {
|
||||
"TRACE": 1,
|
||||
"DEBUG": 5,
|
||||
"INFO": 9,
|
||||
"WARNING": 13,
|
||||
"ERROR": 17,
|
||||
"CRITICAL": 21,
|
||||
}
|
||||
|
||||
# Which process this is. Set by configure_logging; every line carries it so one Loki/DB query can
|
||||
# separate the API's view of a job from the worker's.
|
||||
_source = "api"
|
||||
|
||||
|
||||
# --- correlation context -------------------------------------------------------------------
|
||||
# contextvars, like quota's actor attribution (app/quota.py): set at an entry point, read by the
|
||||
# formatter, never threaded through call signatures. A child task copies the context at creation,
|
||||
# so anything set before `await call_next(...)` is visible to the whole request.
|
||||
_request_id: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"log_request_id", default=None
|
||||
)
|
||||
_user_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_user_id", default=None)
|
||||
_debug_session: contextvars.ContextVar[str | None] = contextvars.ContextVar(
|
||||
"log_debug_session", default=None
|
||||
)
|
||||
_job_id: contextvars.ContextVar[int | None] = contextvars.ContextVar("log_job_id", default=None)
|
||||
|
||||
_CONTEXT_VARS = {
|
||||
"request_id": _request_id,
|
||||
"user_id": _user_id,
|
||||
"debug_session_id": _debug_session,
|
||||
"job_id": _job_id,
|
||||
}
|
||||
|
||||
|
||||
class _RequestScope:
|
||||
"""What the auth dependency learns about a request, for every line logged after it.
|
||||
|
||||
A MUTABLE object behind a contextvar, not plain contextvars, because of how a request actually
|
||||
runs: FastAPI resolves a sync dependency (`auth.current_user`) in a threadpool on a COPY of the
|
||||
context, so a `ContextVar.set()` there is gone by the time the handler runs — measured: that is
|
||||
how the account never reached a handler's own lines. `log_requests` installs one scope per
|
||||
request; every copied context shares the same object, so what the dependency writes is seen by
|
||||
the handler and by the request line itself."""
|
||||
|
||||
__slots__ = ("user_id", "level", "debug_session_id")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.user_id: int | None = None
|
||||
# The user's consented diagnostics level as a logging number; None at the base level.
|
||||
self.level: int | None = None
|
||||
# The user's own open debug session this request named (E16), else None.
|
||||
self.debug_session_id: str | None = None
|
||||
|
||||
|
||||
_scope: contextvars.ContextVar[_RequestScope | None] = contextvars.ContextVar(
|
||||
"log_request_scope", default=None
|
||||
)
|
||||
|
||||
|
||||
def set_user(
|
||||
user_id: int | None,
|
||||
level: str = "INFO",
|
||||
until: datetime | None = None,
|
||||
debug_session: str | None = None,
|
||||
) -> None:
|
||||
"""Attribute the rest of this request's log lines to `user_id`, and log them at the user's
|
||||
consented diagnostics `level` until `until` (E22 S5; "INFO" = no elevation). Called from the
|
||||
auth dependency — the one place the session is actually resolved into an account — rather than
|
||||
from the request middleware, which runs OUTSIDE the session middleware.
|
||||
|
||||
`debug_session` tags every line with the user's open debug session (E16). The caller has
|
||||
already checked it is theirs and open; a session's level arrives here as `level`/`until`.
|
||||
|
||||
The root logger's floor is tracked per elevation: a request without the session's header (an
|
||||
`<img>`, a `<video>` — the browser sends those itself) carries only the user's own level, and
|
||||
must not lower the floor a concurrent request of the session still needs."""
|
||||
_user_id.set(user_id)
|
||||
scope = _scope.get()
|
||||
if scope is None: # outside log_requests (a script, a unit test): this context only
|
||||
scope = _RequestScope()
|
||||
_scope.set(scope)
|
||||
elevated = level != "INFO" and until is not None and until > datetime.now(timezone.utc)
|
||||
scope.user_id = user_id
|
||||
scope.debug_session_id = debug_session
|
||||
scope.level = resolve_level(level) if elevated else None
|
||||
if user_id is not None:
|
||||
key = user_id if debug_session is None else (user_id, debug_session)
|
||||
_track_elevation(key, scope.level, until)
|
||||
|
||||
|
||||
def set_job(job_id: int | None) -> None:
|
||||
"""Attribute this process's log lines to a download job — for the download CHILD, whose whole
|
||||
life is one job (the worker uses `bind()` per job instead, since it juggles several)."""
|
||||
_job_id.set(job_id)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def bind(**fields: Any):
|
||||
"""Bind correlation fields for the duration of a block (a worker job, a scheduler run):
|
||||
|
||||
with logs.bind(job_id=job.id):
|
||||
...
|
||||
|
||||
Unknown field names are ignored rather than raising — a caller binding a field this module does
|
||||
not know about is a typo, not a reason to take down a download."""
|
||||
tokens = [(_CONTEXT_VARS[k].set(v)) for k, v in fields.items() if k in _CONTEXT_VARS]
|
||||
keys = [k for k in fields if k in _CONTEXT_VARS]
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for key, token in zip(keys, tokens):
|
||||
_CONTEXT_VARS[key].reset(token)
|
||||
|
||||
|
||||
def event(code: str, /, **attrs: Any) -> dict[str, Any]:
|
||||
"""`extra=` payload for a log call: a stable dotted event code plus structured attributes.
|
||||
|
||||
Returns a plain dict so it composes with stdlib logging and costs nothing when the level is off
|
||||
(the dict is built either way; keep the attrs cheap — no DB reads inside the call).
|
||||
|
||||
`code` is positional-only so that no attribute name is reserved: `event("x", code=...)` means
|
||||
the attribute, not this parameter. Without the `/` it was a TypeError, and `code` is exactly
|
||||
the word a caller reaches for when the thing that failed has one of its own."""
|
||||
return {"event": code, "attrs": attrs}
|
||||
|
||||
|
||||
# --- redaction -----------------------------------------------------------------------------
|
||||
# Substring matching, not exact keys: the same secret travels as `password`, `current_password`,
|
||||
# `new_password`, `smtp_password`, `plex_server_token`, `access_token`, … Matching on the PART is
|
||||
# what makes a key we have never seen safe by default.
|
||||
_SECRET_KEY_PARTS = (
|
||||
"password",
|
||||
"passphrase",
|
||||
"token",
|
||||
"secret",
|
||||
"cookie",
|
||||
"authorization",
|
||||
"auth_header",
|
||||
"ciphertext",
|
||||
"private_key",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"credential",
|
||||
"session_key",
|
||||
)
|
||||
|
||||
# Query/fragment parameters that carry a credential in a URL we log (OAuth `code`/`state`, our own
|
||||
# `?token=` setup and share links, `#verify=` / `#reset=` fragments).
|
||||
_SECRET_PARAM_NAMES = (
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"id_token",
|
||||
"token",
|
||||
"code",
|
||||
"state",
|
||||
"key",
|
||||
"secret",
|
||||
"password",
|
||||
"pwd",
|
||||
"sig",
|
||||
"signature",
|
||||
"verify",
|
||||
"reset",
|
||||
"setup",
|
||||
)
|
||||
_SECRET_PARAM_RE = re.compile(
|
||||
r"\b((?:" + "|".join(_SECRET_PARAM_NAMES) + r")=)([^&\s\"']+)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Deliberately loose: over-matching redacts a non-address that looks like one (harmless), while
|
||||
# under-matching puts a real address in a log we attach to an issue (not harmless).
|
||||
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
|
||||
REDACTED = "[redacted]"
|
||||
# `extra={PERSIST: False}` keeps a line out of the `log_events` table while stdout still gets it —
|
||||
# how `log_requests` caps anonymous traffic per client (app/logsink.py reads it).
|
||||
PERSIST = "persist"
|
||||
REDACTED_EMAIL = "[email]"
|
||||
|
||||
|
||||
def redact_text(text: str) -> str:
|
||||
"""Scrub a free-text string: credential-bearing URL params and email addresses."""
|
||||
scrubbed = _SECRET_PARAM_RE.sub(lambda m: f"{m.group(1)}{REDACTED}", text)
|
||||
return _EMAIL_RE.sub(REDACTED_EMAIL, scrubbed)
|
||||
|
||||
|
||||
def redact_query(query: str) -> str:
|
||||
"""Scrub a raw query string. DECODED first: the text rules match what was typed, and a browser
|
||||
sends it percent-encoded — `someone%40example.com` is an address `_EMAIL_RE` cannot see. Each
|
||||
parameter also goes through the key rules, so `?new_password=` goes as `redact` would drop it."""
|
||||
pairs = parse_qsl(query, keep_blank_values=True)
|
||||
return "&".join(f"{key}={_redact_param(key, value)}" for key, value in pairs)
|
||||
|
||||
|
||||
def _redact_param(key: str, value: str) -> str:
|
||||
"""One decoded `key=value` pair. The parameter-NAME rule is applied to the key here rather than
|
||||
left to `_SECRET_PARAM_RE` over the rebuilt pair: that pattern was written for the raw,
|
||||
percent-encoded query, where a value holds no literal space, `&` or quote, and its value class
|
||||
stops at the first one. On decoded text `?state=a%20b` would keep its tail and `?code=%20abc`
|
||||
would not match at all — the whole credential straight into the line."""
|
||||
if key.lower() in _SECRET_PARAM_NAMES:
|
||||
return REDACTED
|
||||
return str(redact(value, key=key))
|
||||
|
||||
|
||||
def redact(value: Any, *, key: str = "") -> Any:
|
||||
"""Recursively scrub a structure destined for the log.
|
||||
|
||||
A value whose KEY names a secret is dropped whole (we never log a prefix of a token — a prefix
|
||||
is still a leak and still useless). Everything else is scrubbed as text. Depth is bounded by
|
||||
the structures we actually log (attrs dicts), and non-JSON types fall through to `str`."""
|
||||
lowered = key.lower()
|
||||
if key and any(part in lowered for part in _SECRET_KEY_PARTS):
|
||||
return REDACTED
|
||||
if isinstance(value, dict):
|
||||
return {k: redact(v, key=str(k)) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [redact(v, key=key) for v in value]
|
||||
if isinstance(value, str):
|
||||
return redact_text(value)
|
||||
return value
|
||||
|
||||
|
||||
# --- formatters ----------------------------------------------------------------------------
|
||||
_SCRUBBED_STACK = "scrubbed_stack"
|
||||
|
||||
|
||||
def scrubbed_stack(exc: BaseException, scrub: Callable[[BaseException], str]) -> dict[str, str]:
|
||||
"""`extra=` fragment: `exc`'s traceback, rendered where an `exc_info` stack goes, with the
|
||||
message of every exception in it put through `scrub`.
|
||||
|
||||
For a failure whose message can name someone's data (a download's URL, a file path, a video
|
||||
id). The frames name code and are kept whole; the text at the foot of each link is the data.
|
||||
A plain `exc_info` renders the chain too — the `__cause__`/`__context__` a DB error raised
|
||||
inside `except DownloadError` carries — so scrubbing only the line's own message is not
|
||||
enough. Merge it into the call's `extra`: `{**event(...), **scrubbed_stack(...)}`."""
|
||||
links: list[BaseException] = []
|
||||
link: BaseException | None = exc
|
||||
while link is not None and all(link is not seen for seen in links):
|
||||
links.append(link)
|
||||
link = link.__cause__ or (None if link.__suppress_context__ else link.__context__)
|
||||
parts: list[str] = []
|
||||
for i in range(len(links) - 1, -1, -1): # the oldest first, as Python prints a chain
|
||||
link = links[i]
|
||||
kind = type(link)
|
||||
name = kind.__qualname__
|
||||
if kind.__module__ not in ("builtins", "__main__"):
|
||||
name = f"{kind.__module__}.{name}"
|
||||
try:
|
||||
message = scrub(link)
|
||||
# silent-ok: the message is what could not be rendered, and the placeholder says so — the
|
||||
# caller is in the middle of reporting one failure and must not get a second from here.
|
||||
except Exception: # noqa: BLE001 — a `__str__` that raises, of any shape
|
||||
message = "<unprintable>"
|
||||
frames = "".join(traceback.format_tb(link.__traceback__))
|
||||
head = f"Traceback (most recent call last):\n{frames}" if frames else ""
|
||||
parts.append(f"{head}{name}: {message}" if message else f"{head}{name}")
|
||||
if i:
|
||||
parts.append(
|
||||
"\nThe above exception was the direct cause of the following exception:\n"
|
||||
if links[i - 1].__cause__ is link
|
||||
else "\nDuring handling of the above exception, another exception occurred:\n"
|
||||
)
|
||||
return {_SCRUBBED_STACK: "\n".join(parts)}
|
||||
|
||||
|
||||
# Attributes every LogRecord carries; anything else a caller passed via `extra=` is a structured
|
||||
# attribute and is merged into `attrs` (so `extra={"job_id": 7}` works as well as `event(...)`).
|
||||
_RECORD_STANDARD = frozenset(
|
||||
vars(logging.LogRecord("", 0, "", 0, "", None, None)).keys()
|
||||
| {"message", "asctime", "taskName", "event", "attrs", PERSIST, "elevated", _SCRUBBED_STACK}
|
||||
)
|
||||
|
||||
|
||||
class ContextFilter(logging.Filter):
|
||||
"""Copies the correlation contextvars onto the record. A Filter (not the formatter) so the
|
||||
values are captured at emit time, before a QueueHandler can move the record to another
|
||||
thread — the DB sink in S3 depends on that."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
for name, var in _CONTEXT_VARS.items():
|
||||
value = var.get()
|
||||
if value is not None and not hasattr(record, name):
|
||||
setattr(record, name, value)
|
||||
scope = _scope.get()
|
||||
if scope is not None and scope.user_id is not None and not hasattr(record, "user_id"):
|
||||
record.user_id = scope.user_id
|
||||
if (
|
||||
scope is not None
|
||||
and scope.debug_session_id is not None
|
||||
and not hasattr(record, "debug_session_id")
|
||||
):
|
||||
record.debug_session_id = scope.debug_session_id
|
||||
return True
|
||||
|
||||
|
||||
class LevelGate(logging.Filter):
|
||||
"""The configured level, enforced at the HANDLER, with one exception: the lines of a request
|
||||
whose user raised their diagnostics level (E22 S5).
|
||||
|
||||
The root logger's level cannot do this alone: it is one number for the whole process, and the
|
||||
point is to log ONE user's requests in detail while everyone else's stay at the base level. So
|
||||
the root is lowered to the lowest level any active elevation needs (`_track_elevation`), and
|
||||
this gate drops what only an elevation let through unless the line belongs to that elevated
|
||||
request. It also marks such a line `elevated`, which is what makes the `log_events` sink keep
|
||||
every line of that request, not just its warnings."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
scope = _scope.get()
|
||||
if scope is not None and scope.level is not None:
|
||||
record.elevated = True
|
||||
return record.levelno >= min(scope.level, _base_level)
|
||||
return record.levelno >= _base_level
|
||||
|
||||
|
||||
def _record_attrs(record: logging.LogRecord) -> dict[str, Any]:
|
||||
extra = {k: v for k, v in vars(record).items() if k not in _RECORD_STANDARD}
|
||||
attrs = dict(getattr(record, "attrs", None) or {})
|
||||
attrs.update({k: v for k, v in extra.items() if k not in _CONTEXT_VARS})
|
||||
return attrs
|
||||
|
||||
|
||||
def _safe_message(record: logging.LogRecord) -> str:
|
||||
"""`record.getMessage()` raises if the args don't match the format string. A broken log call
|
||||
must not take down the caller, so the failure is reported IN the line instead."""
|
||||
try:
|
||||
return record.getMessage()
|
||||
# silent-ok: the failure is reported IN the returned line. Logging it would re-enter this same
|
||||
# formatter with a record built out of the one that just failed.
|
||||
except Exception as exc: # noqa: BLE001 — any formatting error, reported not raised
|
||||
return f"{record.msg!r} (log formatting failed: {exc})"
|
||||
|
||||
|
||||
def record_fields(record: logging.LogRecord) -> dict[str, Any]:
|
||||
"""The event as structured fields, REDACTED — the one definition both the stdout line and the
|
||||
`log_events` row are built from, so the two sinks can never disagree about what is safe to keep.
|
||||
Absent optional fields are left out; `ts` is an aware datetime."""
|
||||
fields: dict[str, Any] = {
|
||||
"ts": datetime.fromtimestamp(record.created, timezone.utc),
|
||||
"level": record.levelname,
|
||||
"sev": SEVERITY.get(record.levelname, 9),
|
||||
"source": _source,
|
||||
"logger": record.name,
|
||||
"msg": redact_text(_safe_message(record)),
|
||||
}
|
||||
code = getattr(record, "event", None)
|
||||
if code:
|
||||
fields["event"] = code
|
||||
for name in _CONTEXT_VARS:
|
||||
value = getattr(record, name, None)
|
||||
if value is not None:
|
||||
fields[name] = value
|
||||
attrs = _record_attrs(record)
|
||||
if attrs:
|
||||
fields["attrs"] = redact(attrs)
|
||||
if record.exc_info:
|
||||
fields["exc"] = redact_text(logging.Formatter().formatException(record.exc_info))
|
||||
elif getattr(record, _SCRUBBED_STACK, None):
|
||||
fields["exc"] = redact_text(getattr(record, _SCRUBBED_STACK))
|
||||
elif record.exc_text:
|
||||
# A QueueHandler-style hand-off formats the traceback into exc_text and drops exc_info.
|
||||
fields["exc"] = redact_text(record.exc_text)
|
||||
fields["version"] = settings.app_version
|
||||
return fields
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
"""One JSON object per line — the shape Loki and the `log_events` table both ingest."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload = record_fields(record)
|
||||
payload["ts"] = payload["ts"].isoformat(timespec="milliseconds")
|
||||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
class TextFormatter(logging.Formatter):
|
||||
"""Human-readable single line for a terminal (`LOG_FORMAT=text`): the same content, ordered for
|
||||
reading. Redaction applies here too — a dev log is still a log someone pastes into an issue."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
stamp = datetime.fromtimestamp(record.created, timezone.utc).strftime("%H:%M:%S.%f")[:-3]
|
||||
head = f"{stamp} {record.levelname:<5} [{record.name}]"
|
||||
rid = getattr(record, "request_id", None)
|
||||
if rid:
|
||||
head += f" {rid}"
|
||||
code = getattr(record, "event", None)
|
||||
line = f"{head} {code + ' — ' if code else ''}{redact_text(_safe_message(record))}"
|
||||
attrs = _record_attrs(record)
|
||||
if attrs:
|
||||
line += f" {json.dumps(redact(attrs), ensure_ascii=False, default=str)}"
|
||||
if record.exc_info:
|
||||
line += "\n" + redact_text(self.formatException(record.exc_info))
|
||||
elif getattr(record, _SCRUBBED_STACK, None):
|
||||
line += "\n" + redact_text(getattr(record, _SCRUBBED_STACK))
|
||||
return line
|
||||
|
||||
|
||||
# --- configuration -------------------------------------------------------------------------
|
||||
# Third-party loggers that are noisy at our own level. Pinned explicitly so the diagnostics knob can
|
||||
# lower the ROOT level to DEBUG/TRACE without drowning the output in SQL statements and HTTP chatter
|
||||
# — raising THESE is a deliberate, separate act (E22.6's console).
|
||||
_THIRD_PARTY_LEVELS = {
|
||||
"sqlalchemy": "WARNING",
|
||||
"sqlalchemy.engine": "WARNING",
|
||||
"apscheduler": "WARNING",
|
||||
"urllib3": "WARNING",
|
||||
"httpx": "WARNING",
|
||||
"httpcore": "WARNING",
|
||||
"asyncio": "WARNING",
|
||||
"multipart": "WARNING",
|
||||
"PIL": "WARNING",
|
||||
"alembic": "INFO",
|
||||
# uvicorn's own startup/shutdown/WebSocket lines are worth keeping; its ACCESS log is not —
|
||||
# it is one line per request with the query string in it, replaced by `log_requests` below.
|
||||
"uvicorn": "INFO",
|
||||
"uvicorn.error": "INFO",
|
||||
"uvicorn.access": "CRITICAL",
|
||||
}
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""The logger for one module: `logs.get_logger(__name__)` in `app/plex/sync.py` is
|
||||
`siftlode.plex.sync`.
|
||||
|
||||
The names were literals before, and 25 of them covered 34 modules — six different Plex modules
|
||||
all called themselves `siftlode.plex`, so neither a level pinned per logger nor E22.6's console
|
||||
filter could tell a sync line from a stream line. Derived from `__name__` there is exactly one
|
||||
name per module, it cannot drift from the file it sits in, and `test_logger_names.py` fails if a
|
||||
module hand-rolls one again.
|
||||
|
||||
`__main__` is resolved to the module it really is. The worker runs as `python -m app.worker`
|
||||
and spawns `python -m app.downloads.runner`, so in the two long-running processes `__name__`
|
||||
is `__main__` — taken at its word, both would log as the bare root. `-m` records the real name
|
||||
on the module's spec; a plain script has none and stays the root."""
|
||||
if name == "__main__":
|
||||
spec = getattr(sys.modules.get("__main__"), "__spec__", None)
|
||||
name = getattr(spec, "name", None) or name
|
||||
stem = name.removeprefix("app.") if name.startswith("app.") else ""
|
||||
# logger-name-ok: this call IS the derivation
|
||||
return logging.getLogger(f"siftlode.{stem}" if stem else "siftlode")
|
||||
|
||||
|
||||
def resolve_level(name: str) -> int:
|
||||
"""Level name → number. Both ways in (the `LOG_LEVEL` env and the `log_level` admin knob) are
|
||||
already held to the `LogLevel` choices, so an unknown name here is a programming error."""
|
||||
value = logging.getLevelName(name)
|
||||
if not isinstance(value, int):
|
||||
raise ValueError(f"unknown log level {name!r}")
|
||||
return value
|
||||
|
||||
|
||||
def configure_logging(source: str) -> None:
|
||||
"""Configure logging for THIS process. Call once, first thing, in every entry point:
|
||||
`main.py` (api), `worker.main`, `downloads/runner.main`, `alembic/env.py`.
|
||||
|
||||
`source` names the process in every line. The level starts at `LOG_LEVEL` (the bootstrap default);
|
||||
the runtime admin knob goes through `set_level()` instead, so changing it needs no restart.
|
||||
|
||||
Our loggers deliberately keep `propagate=True` and own no handler: everything reaches the single
|
||||
root StreamHandler. That is what keeps `caplog` working in tests, and it means a new logger name
|
||||
is configured the moment it is created."""
|
||||
global _source
|
||||
_source = source
|
||||
formatter = settings.log_format
|
||||
logging.config.dictConfig(
|
||||
{
|
||||
"version": 1,
|
||||
# Never disable loggers created at import time (module-level `getLogger` calls run
|
||||
# before this function does).
|
||||
"disable_existing_loggers": False,
|
||||
"formatters": {
|
||||
"json": {"()": f"{__name__}.JsonFormatter"},
|
||||
"text": {"()": f"{__name__}.TextFormatter"},
|
||||
},
|
||||
"filters": {
|
||||
"context": {"()": f"{__name__}.ContextFilter"},
|
||||
"gate": {"()": f"{__name__}.LevelGate"},
|
||||
},
|
||||
"handlers": {
|
||||
"stdout": {
|
||||
"class": "logging.StreamHandler",
|
||||
"stream": sys.stdout,
|
||||
"formatter": formatter,
|
||||
"filters": ["context", "gate"],
|
||||
}
|
||||
},
|
||||
"root": {"level": resolve_level(settings.log_level), "handlers": ["stdout"]},
|
||||
"loggers": {name: {"level": lvl} for name, lvl in _THIRD_PARTY_LEVELS.items()},
|
||||
}
|
||||
)
|
||||
set_level(settings.log_level)
|
||||
|
||||
|
||||
def set_level(name: str) -> None:
|
||||
"""Change the running process's level (the `log_level` admin knob). Third-party loggers keep
|
||||
their pinned levels, so turning on DEBUG shows OUR detail, not SQLAlchemy's."""
|
||||
global _base_level
|
||||
_base_level = resolve_level(name)
|
||||
_apply_root_level(force=True)
|
||||
|
||||
|
||||
def current_level() -> str:
|
||||
"""The configured level — not the root logger's, which an active elevation may have lowered."""
|
||||
return logging.getLevelName(_base_level or logging.getLogger().getEffectiveLevel())
|
||||
|
||||
|
||||
# --- per-user elevation (E22 S5) -----------------------------------------------------------
|
||||
# The base level (`LOG_LEVEL`, then the admin knob). NOTSET until configure_logging runs — a unit
|
||||
# test, a script — so the gate lets everything through there, as it did before the gate existed.
|
||||
_base_level = logging.NOTSET
|
||||
# → (level number, expiry) for every elevation this process has seen and not yet expired: a user's
|
||||
# own, by user id, and a debug session's (E16), by (user id, session id) — its requests alone carry
|
||||
# its level, so it is tracked apart from the user's own requests.
|
||||
_ElevationKey = int | tuple[int, str]
|
||||
_elevations: dict[_ElevationKey, tuple[int, datetime]] = {}
|
||||
_elevations_lock = threading.Lock()
|
||||
_root_applied: int | None = None
|
||||
|
||||
|
||||
def _track_elevation(key: _ElevationKey, level: int | None, until: datetime | None) -> None:
|
||||
with _elevations_lock:
|
||||
if level is None or until is None:
|
||||
_elevations.pop(key, None)
|
||||
else:
|
||||
_elevations[key] = (level, until)
|
||||
active = bool(_elevations)
|
||||
if active or _root_applied != _base_level:
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def _apply_root_level(force: bool = False) -> None:
|
||||
"""Root level = the lowest of the base and every unexpired elevation (expired ones are pruned
|
||||
here). `setLevel` clears every logger's level cache, so it runs only when the number changes."""
|
||||
global _root_applied
|
||||
if _base_level == logging.NOTSET:
|
||||
return # not configured: the root keeps whatever the process set up
|
||||
now = datetime.now(timezone.utc)
|
||||
# The level is SET under the lock too, not just computed: two threads that each computed a
|
||||
# different answer would otherwise apply them in either order, and the older one could win —
|
||||
# the root back at the base while an elevation is still active.
|
||||
with _elevations_lock:
|
||||
for key in [k for k, (_, until) in _elevations.items() if until <= now]:
|
||||
del _elevations[key]
|
||||
lowest = min([_base_level, *(lvl for lvl, _ in _elevations.values())])
|
||||
if force or lowest != _root_applied:
|
||||
logging.getLogger().setLevel(lowest)
|
||||
_root_applied = lowest
|
||||
|
||||
|
||||
# --- erased accounts ------------------------------------------------------------------------
|
||||
# `auth.purge_user` deletes an account's `log_events` rows, but a line about that account can still
|
||||
# be on its way: the erasing request's own access line is logged AFTER the handler committed, a queued
|
||||
# row waits for the sink's writer, a background task logs later. Those rows are stored without the
|
||||
# account (`logsink` asks here). Remembered for an hour in this process — a line later than that has
|
||||
# no request left to come from, and the retention job is the backstop for any other process.
|
||||
ERASED_MEMORY_SECONDS = 3600.0
|
||||
_erased: dict[int, float] = {}
|
||||
_erased_lock = threading.Lock()
|
||||
# Held by whoever checks rows against `is_erased` AND writes them (`erasure_guard`). `forget_user`
|
||||
# takes it too, so an erasure waits for a write that already passed the check to commit — and the
|
||||
# purge's DELETE, which runs after `forget_user`, then removes those rows as well. Without it a row
|
||||
# checked a moment before the erasure could be inserted a moment after the DELETE.
|
||||
_erasure_guard = threading.Lock()
|
||||
|
||||
|
||||
def erasure_guard() -> threading.Lock:
|
||||
"""Hold this around "drop erased accounts from these rows, insert them, commit"."""
|
||||
return _erasure_guard
|
||||
|
||||
|
||||
def forget_user(user_id: int) -> None:
|
||||
"""Mark `user_id` erased: no line logged about it from now on is stored under its id. Call BEFORE
|
||||
deleting its rows (see `_erasure_guard`)."""
|
||||
now = time.monotonic()
|
||||
with _erasure_guard, _erased_lock:
|
||||
for stale in [u for u, at in _erased.items() if now - at > ERASED_MEMORY_SECONDS]:
|
||||
del _erased[stale]
|
||||
_erased[user_id] = now
|
||||
with _elevations_lock:
|
||||
for key in [
|
||||
k for k in _elevations if k == user_id or (isinstance(k, tuple) and k[0] == user_id)
|
||||
]:
|
||||
del _elevations[key]
|
||||
# That pop can have removed the lowest elevation, and the root level is the lowest of the base
|
||||
# and every unexpired one. Recomputed here rather than left to whichever later request happens
|
||||
# to call `_track_elevation` — the invariant `_apply_root_level` documents holds at every pop.
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def end_session_elevation(session_id: str) -> None:
|
||||
"""A debug session was stopped: its elevation ends now, not at its cap."""
|
||||
with _elevations_lock:
|
||||
for key in [k for k in _elevations if isinstance(k, tuple) and k[1] == session_id]:
|
||||
del _elevations[key]
|
||||
_apply_root_level()
|
||||
|
||||
|
||||
def is_erased(user_id: int | None) -> bool:
|
||||
if user_id is None:
|
||||
return False
|
||||
with _erased_lock:
|
||||
at = _erased.get(user_id)
|
||||
return at is not None and time.monotonic() - at <= ERASED_MEMORY_SECONDS
|
||||
|
||||
|
||||
def current_source() -> str:
|
||||
"""The process name `configure_logging` was given (api, worker, runner, alembic)."""
|
||||
return _source
|
||||
|
||||
|
||||
# --- request logging -----------------------------------------------------------------------
|
||||
# A client may supply its own request id so a browser log line and a server line join up (E22 S4).
|
||||
# Accepted only in this shape: it ends up in a log field, and an unbounded header would be a way to
|
||||
# write arbitrary text into the log.
|
||||
_REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,64}$")
|
||||
|
||||
# Paths whose success is logged at DEBUG rather than INFO: the container healthcheck runs every 15s
|
||||
# (5.7k lines/day saying nothing), hashed static assets are served by the hundred per page load, and
|
||||
# the admin log console reads the stored log — its live tail every few seconds, so at INFO the log
|
||||
# would fill with the console watching itself. A FAILURE on any of them still logs at WARNING/ERROR
|
||||
# — this lowers the floor, it does not hide.
|
||||
_QUIET_PREFIXES = ("/healthz", "/assets/", "/static/", "/api/admin/logs")
|
||||
|
||||
# The reads the frontend POLLS (a `useLiveQuery`, the `me` heartbeat, the sync status, the subtitle
|
||||
# progress), by route template, GET only — logged at DEBUG like the prefixes above. One open tab asked
|
||||
# for the two unread counts, `/api/me` and the download index every 30 s, and at INFO those were ~95%
|
||||
# of the stored log (measured on prod 2026-09-19): 11k lines a day per tab burying every line someone
|
||||
# caused. The same route answered with a failure still logs at WARNING/ERROR.
|
||||
_POLLED_ROUTES = frozenset(
|
||||
{
|
||||
"/api/me",
|
||||
"/api/me/notifications/unread_count",
|
||||
"/api/messages/unread_count",
|
||||
"/api/downloads",
|
||||
"/api/downloads/index",
|
||||
"/api/downloads/usage",
|
||||
"/api/sync/my-status",
|
||||
"/api/admin/scheduler",
|
||||
"/api/admin/downloads",
|
||||
"/api/admin/downloads/storage",
|
||||
"/api/plex/subtitle/{rating_key}/{sub_ord}/status",
|
||||
}
|
||||
)
|
||||
|
||||
# Not `siftlode.logs`: this is the replacement for uvicorn's ACCESS log, and `siftlode.http` is
|
||||
# what an operator pins a level on when they want request lines and nothing else. The module it
|
||||
# happens to live in is not what the name is about.
|
||||
_request_log = logging.getLogger("siftlode.http") # logger-name-ok: named for the traffic
|
||||
|
||||
|
||||
async def log_requests(request, call_next):
|
||||
"""The single access-log line, replacing uvicorn's.
|
||||
|
||||
What changes versus `uvicorn.access`: the ROUTE TEMPLATE instead of the concrete path (so
|
||||
`/api/videos/{video_id}` groups), never the query string (it carried `?account=`, and at TRACE
|
||||
the redacted query belongs in attrs, not in the URL), a request id echoed back in
|
||||
`X-Request-Id` (with the template in `X-Route`), the duration, and the account — read from `request.state`, which the auth
|
||||
dependency fills, because this middleware sits outside both the session middleware and the
|
||||
downstream task's context."""
|
||||
incoming = request.headers.get("x-request-id", "")
|
||||
rid = incoming if _REQUEST_ID_RE.match(incoming) else uuid.uuid4().hex[:16]
|
||||
token = _request_id.set(rid)
|
||||
scope = _RequestScope()
|
||||
scope_token = _scope.set(scope)
|
||||
request.state.request_id = rid
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
# The unhandled-exception path: FastAPI's ServerErrorMiddleware re-raises after its own
|
||||
# 500, so without this the ONE request that most needs a log line is the one with no
|
||||
# access-log entry at all.
|
||||
_request_log.exception(
|
||||
"request failed",
|
||||
extra={
|
||||
**event(
|
||||
"http.request",
|
||||
method=request.method,
|
||||
path=_route_template(request),
|
||||
duration_ms=round((time.perf_counter() - started) * 1000, 1),
|
||||
),
|
||||
**_request_owner(request, logging.ERROR),
|
||||
},
|
||||
)
|
||||
_scope.reset(scope_token)
|
||||
_request_id.reset(token)
|
||||
raise
|
||||
duration_ms = round((time.perf_counter() - started) * 1000, 1)
|
||||
status = response.status_code
|
||||
quiet = request.url.path.startswith(_QUIET_PREFIXES) or (
|
||||
request.method == "GET" and _route_template(request) in _POLLED_ROUTES
|
||||
)
|
||||
if status >= 500:
|
||||
level = logging.ERROR
|
||||
elif status >= 400:
|
||||
level = logging.WARNING
|
||||
else:
|
||||
level = logging.DEBUG if quiet else logging.INFO
|
||||
attrs: dict[str, Any] = {
|
||||
"method": request.method,
|
||||
"path": _route_template(request),
|
||||
"status": status,
|
||||
"duration_ms": duration_ms,
|
||||
}
|
||||
# Teljes (TRACE) the user consented to: the query string too — redacted, and in attrs only, never
|
||||
# in the message.
|
||||
if request.url.query and _traced(scope):
|
||||
attrs["query"] = redact_query(request.url.query)
|
||||
_request_log.log(
|
||||
level,
|
||||
"%s %s %s",
|
||||
request.method,
|
||||
_route_template(request),
|
||||
status,
|
||||
extra={**event("http.request", **attrs), **_request_owner(request, level)},
|
||||
)
|
||||
response.headers["X-Request-Id"] = rid
|
||||
# The browser logs its own copy of this line (frontend lib/http.ts) and must pick the same level:
|
||||
# it learns "this one is quiet" here instead of keeping a second copy of the polled-route list.
|
||||
if level == logging.DEBUG:
|
||||
response.headers["X-Log-Quiet"] = "1"
|
||||
# The template, so the browser's copy of this line groups the same way (frontend lib/http.ts).
|
||||
# Only a MATCHED route's: a template never carries a path parameter's value (a share token stays
|
||||
# `{token}`), while the raw-path fallback of an unmatched request could — the browser then
|
||||
# templates the path itself.
|
||||
matched = _matched_route(request)
|
||||
if matched:
|
||||
response.headers["X-Route"] = matched
|
||||
_scope.reset(scope_token)
|
||||
_request_id.reset(token)
|
||||
return response
|
||||
|
||||
|
||||
def consented_trace() -> bool:
|
||||
"""Whether the current request's OWN user turned Teljes (TRACE) on — the gate for a line that
|
||||
carries what they typed or what was quoted back to them. The level alone is not enough: an
|
||||
admin running the whole process at TRACE would pass it for every user, and a scheduler thread,
|
||||
which has no request, for none of them in particular. Guard such a line with this, then log it
|
||||
at TRACE."""
|
||||
scope = _scope.get()
|
||||
return scope is not None and _traced(scope)
|
||||
|
||||
|
||||
def _traced(scope: _RequestScope) -> bool:
|
||||
"""Whether this request's query string is logged: only under its OWN user's Teljes consent. An
|
||||
admin setting the whole process to TRACE gets more detail, not every user's search terms — the
|
||||
privacy policy promises those are kept only for a user who agreed to it."""
|
||||
return scope.level is not None and scope.level <= TRACE
|
||||
|
||||
|
||||
# Anonymous request lines kept per client per minute. Signed-in traffic is not capped: it is a known
|
||||
# account and exactly what an admin needs to see. Anonymous traffic is where the volume can come from
|
||||
# nobody in particular — a scanner walking `/wp-login.php`, a bot on a share link — and each of its
|
||||
# 404s is a WARNING that would otherwise land in `log_events` without limit. The client key is an
|
||||
# in-memory counter key only; no address is ever written to a log line or a row.
|
||||
ANONYMOUS_LINES_PER_MINUTE = 60
|
||||
_anonymous_lines = RateLimiter(ANONYMOUS_LINES_PER_MINUTE, 60.0)
|
||||
|
||||
|
||||
def _request_owner(request, level: int) -> dict[str, Any]:
|
||||
"""`extra=` fields naming who the request belongs to, and whether its line may be stored. The
|
||||
account comes from `request.state` (filled by the auth dependency), so it lands in the line's
|
||||
top-level `user_id` — the correlation field — rather than in `attrs`."""
|
||||
user_id = getattr(request.state, "user_id", None)
|
||||
if user_id is not None:
|
||||
return {"user_id": user_id}
|
||||
if level < logging.INFO:
|
||||
return {} # a quiet line is never stored anyway — don't spend the client's allowance on it
|
||||
return {} if _anonymous_lines.allow(client_key(request)) else {PERSIST: False}
|
||||
|
||||
|
||||
def _matched_route(request) -> str | None:
|
||||
"""The matched route's template (`/api/videos/{video_id}`), or None when nothing matched."""
|
||||
return getattr(request.scope.get("route"), "path", None)
|
||||
|
||||
|
||||
def _route_template(request) -> str:
|
||||
"""The matched route's template, else the raw path for a 404 or a static file. Never the query
|
||||
string."""
|
||||
return _matched_route(request) or request.url.path
|
||||
@@ -0,0 +1,367 @@
|
||||
"""The `log_events` sink (E22 S3): the stored copy of the log the diagnostics console reads.
|
||||
|
||||
stdout stays the complete log (docker keeps 3×50 MB of it); this table keeps the part worth querying
|
||||
for `log_retention_days`:
|
||||
|
||||
* every WARNING and above,
|
||||
* the `http.request` line of every request at INFO or above (not the DEBUG-level health checks and
|
||||
static files),
|
||||
* every line of a request whose user raised their diagnostics level (E22 S5 — `logs.LevelGate`
|
||||
marks them), which is the detail they consented to have kept,
|
||||
* nothing a caller marked `extra={logs.PERSIST: False}` — how the request log caps anonymous
|
||||
traffic per client.
|
||||
|
||||
What the table must never do is slow down or break the code that logs. So a line is redacted and
|
||||
turned into a row on the CALLER's thread (the same `logs.record_fields` the stdout line is built
|
||||
from), put on a bounded queue without blocking, and written in batches by one background thread
|
||||
on its own connection — never inside a request's transaction. A full queue drops the row and
|
||||
counts it; a failed write drops the batch and says so on stderr. Both are reported rather than
|
||||
retried: a log that retries into a sick database is how a small outage becomes a big one.
|
||||
|
||||
Flood control is the second half. On 2026-09-16 a YouTube RSS outage made prod log 11k near-identical
|
||||
warnings in four hours. stdout keeps them all; the table keeps `REPEATS_PER_MINUTE` of each message
|
||||
template per minute and one `log.repeats_suppressed` row saying how many more there were.
|
||||
|
||||
Core layer: imports `logs`, `db`, `models`, `config` — nothing above them.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, insert
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.config import settings
|
||||
from app.db import engine
|
||||
from app.models import LogEvent
|
||||
from app.utils import iso
|
||||
|
||||
QUEUE_MAX = 10_000
|
||||
BATCH_MAX = 500
|
||||
FLUSH_SECONDS = 1.0
|
||||
REPEATS_PER_MINUTE = 30
|
||||
REPEAT_WINDOW_SECONDS = 60.0
|
||||
# Distinct templates tracked per window. Past this many, a new template is stored without counting
|
||||
# — the limiter must not become the unbounded thing it exists to prevent.
|
||||
_REPEAT_KEYS_MAX = 2_000
|
||||
|
||||
# Column widths from models.LogEvent. A value past its column would fail the WHOLE batch it rides in.
|
||||
_LIMITS = {
|
||||
"level": 10,
|
||||
"source": 16,
|
||||
"logger": 128,
|
||||
"event": 64,
|
||||
"request_id": 64,
|
||||
"debug_session_id": 64,
|
||||
"version": 64,
|
||||
"msg": 8_000,
|
||||
"exc": 32_000,
|
||||
}
|
||||
|
||||
_log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def should_store(record: logging.LogRecord) -> bool:
|
||||
"""Whether this record belongs in `log_events` (see the module docstring)."""
|
||||
if getattr(record, logs.PERSIST, True) is False:
|
||||
return False
|
||||
if record.levelno >= logging.WARNING or getattr(record, "elevated", False):
|
||||
return True
|
||||
return getattr(record, "event", None) == "http.request" and record.levelno >= logging.INFO
|
||||
|
||||
|
||||
def _without_nul(value: Any) -> Any:
|
||||
"""`value` with every NUL character replaced, in keys and nested values alike. Postgres refuses
|
||||
NUL in text and `\\u0000` in JSONB, and a refused row fails the WHOLE batch it rides in — every
|
||||
user's rows with it. Anything logged can carry one (`GET /assets/%00` logs the decoded path)."""
|
||||
if isinstance(value, str):
|
||||
return value.replace("\x00", "�")
|
||||
if isinstance(value, dict):
|
||||
return {_without_nul(k): _without_nul(v) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [_without_nul(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def to_row(fields: dict[str, Any]) -> dict[str, Any]:
|
||||
"""`logs.record_fields` output → a `log_events` row: known columns only, text clipped to its
|
||||
column and free of NUL, attrs made JSON-safe (anything exotic becomes its `str`, as on stdout)."""
|
||||
row = {
|
||||
column: fields.get(column) for column in LogEvent.__table__.columns.keys() if column != "id"
|
||||
}
|
||||
for column, limit in _LIMITS.items():
|
||||
value = row.get(column)
|
||||
if isinstance(value, str):
|
||||
value = _without_nul(value)
|
||||
row[column] = value[: limit - 1] + "…" if len(value) > limit else value
|
||||
if row["attrs"] is not None:
|
||||
row["attrs"] = _without_nul(
|
||||
json.loads(json.dumps(row["attrs"], ensure_ascii=False, default=str))
|
||||
)
|
||||
return row
|
||||
|
||||
|
||||
def without_erased(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""The rows, with the account taken off any whose account was erased meanwhile
|
||||
(`logs.forget_user`). Checked right before the INSERT, under `logs.erasure_guard()` held until
|
||||
the commit: a row built before the erasure can still be waiting for its batch."""
|
||||
for row in rows:
|
||||
if logs.is_erased(row.get("user_id")):
|
||||
row["user_id"] = None
|
||||
return rows
|
||||
|
||||
|
||||
def _notice(code: str, msg: str, **attrs: Any) -> dict[str, Any]:
|
||||
"""A row the sink writes about ITSELF. Built directly, not logged: logging it would route it back
|
||||
through this sink (and the writer thread's own records are deliberately ignored)."""
|
||||
return to_row(
|
||||
{
|
||||
"ts": datetime.now(timezone.utc),
|
||||
"level": "WARNING",
|
||||
"sev": logs.SEVERITY["WARNING"],
|
||||
"source": logs.current_source(),
|
||||
"logger": _log.name,
|
||||
"event": code,
|
||||
"msg": msg,
|
||||
"attrs": attrs,
|
||||
"version": settings.app_version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
_DIGITS = re.compile(r"\d+")
|
||||
|
||||
|
||||
def _template(record: logging.LogRecord) -> str:
|
||||
"""The flood limiter's grouping key: the message template. Except when the template is a bare
|
||||
`%s` — a pass-through for someone else's text (yt-dlp's lines, `formats._YdlLog`), where the
|
||||
template says nothing and every distinct line would share one bucket. Then the text itself
|
||||
groups, with its numbers folded, so `Retrying fragment 5 (1/10)` and `(2/10)` are still one
|
||||
repeat and a different warning is not crowded out by them."""
|
||||
msg = str(record.msg)
|
||||
if msg != "%s" or not record.args:
|
||||
return msg
|
||||
return _DIGITS.sub("#", record.getMessage())[:200]
|
||||
|
||||
|
||||
class _Repeats:
|
||||
"""Per-template admission for one minute at a time. Used by the writer thread only (unlocked)."""
|
||||
|
||||
def __init__(self, per_window: int, window_seconds: float):
|
||||
self.per_window = per_window
|
||||
self.window = window_seconds
|
||||
self.started = time.monotonic()
|
||||
self.seen: dict[tuple[str, str, int], int] = {}
|
||||
|
||||
def admit(self, key: tuple[str, str, int] | None) -> bool:
|
||||
if key is None:
|
||||
return True
|
||||
count = self.seen.get(key)
|
||||
if count is None and len(self.seen) >= _REPEAT_KEYS_MAX:
|
||||
return True
|
||||
self.seen[key] = (count or 0) + 1
|
||||
return self.seen[key] <= self.per_window
|
||||
|
||||
def roll(self, now: float, *, final: bool = False) -> list[dict[str, Any]]:
|
||||
"""Close the window if it is over — or, `final`, because the process is: one notice per
|
||||
template that went past the limit. Without `final` a process that lives under a minute
|
||||
(every download child) would drop its repeats and never say so."""
|
||||
if not final and now - self.started < self.window:
|
||||
return []
|
||||
notices = [
|
||||
_notice(
|
||||
"log.repeats_suppressed",
|
||||
f"{count - self.per_window} more like this were not stored",
|
||||
logger=logger,
|
||||
template=logs.redact_text(template),
|
||||
suppressed=count - self.per_window,
|
||||
window_seconds=int(self.window),
|
||||
)
|
||||
for (logger, template, _level), count in self.seen.items()
|
||||
if count > self.per_window
|
||||
]
|
||||
self.seen.clear()
|
||||
self.started = now
|
||||
return notices
|
||||
|
||||
|
||||
class LogSink(logging.Handler):
|
||||
"""The handler plus its writer. `start()` runs the writer thread; a test drives `write_pending()`
|
||||
directly instead, on its own thread, with no timing involved."""
|
||||
|
||||
def __init__(self, bind: Engine = engine):
|
||||
super().__init__(level=logging.NOTSET)
|
||||
# Correlation fields are contextvars: capture them now, on the thread that logged.
|
||||
self.addFilter(logs.ContextFilter())
|
||||
self._bind = bind
|
||||
self._queue: queue.Queue[tuple[tuple[str, str, int] | None, dict[str, Any]]] = queue.Queue(
|
||||
QUEUE_MAX
|
||||
)
|
||||
self._dropped = 0
|
||||
self._dropped_lock = threading.Lock()
|
||||
self._repeats = _Repeats(REPEATS_PER_MINUTE, REPEAT_WINDOW_SECONDS)
|
||||
self._stopping = threading.Event()
|
||||
self._writer: threading.Thread | None = None
|
||||
|
||||
# --- caller side ---------------------------------------------------------------------------
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
if threading.current_thread() is self._writer:
|
||||
return # the writer's own trouble (a driver warning mid-insert) must not feed itself
|
||||
if not should_store(record):
|
||||
return
|
||||
try:
|
||||
row = to_row(logs.record_fields(record))
|
||||
# silent-ok: `handleError` is the report — logging's own out-of-band channel to stderr.
|
||||
# A log call here would be handed to this handler, which is the one that just failed.
|
||||
except Exception: # noqa: BLE001 — a record we cannot turn into a row is reported, not raised
|
||||
self.handleError(record)
|
||||
return
|
||||
# Request lines are one per request by nature; only the repeated SAME message is a flood.
|
||||
key = (
|
||||
None
|
||||
if row["event"] == "http.request"
|
||||
else (record.name, _template(record), record.levelno)
|
||||
)
|
||||
try:
|
||||
self._queue.put_nowait((key, row))
|
||||
# silent-ok: counted, not logged — the count goes out as the `log.sink.dropped` notice row
|
||||
# the writer emits, which is the one report that does not feed this queue again.
|
||||
except queue.Full:
|
||||
with self._dropped_lock:
|
||||
self._dropped += 1
|
||||
|
||||
# --- writer side ---------------------------------------------------------------------------
|
||||
def write_pending(self, wait: float = 0.0, *, final: bool = False) -> int:
|
||||
"""Write one batch: whatever is queued (up to BATCH_MAX), waiting up to `wait` seconds for the
|
||||
first item. Returns the number of rows written, notices included. `final` closes the
|
||||
repeat window early — the last batch before the process exits."""
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
item = self._queue.get(timeout=wait) if wait > 0 else self._queue.get_nowait()
|
||||
# silent-ok: an idle sink. Nothing failed; there was simply nothing waiting.
|
||||
except queue.Empty:
|
||||
item = None
|
||||
while item is not None:
|
||||
key, row = item
|
||||
if self._repeats.admit(key):
|
||||
rows.append(row)
|
||||
if len(rows) >= BATCH_MAX:
|
||||
break
|
||||
try:
|
||||
item = self._queue.get_nowait()
|
||||
# silent-ok: the batch ends here because the queue ran dry, which is the normal end.
|
||||
except queue.Empty:
|
||||
item = None
|
||||
rows.extend(self._repeats.roll(time.monotonic(), final=final))
|
||||
with self._dropped_lock:
|
||||
dropped, self._dropped = self._dropped, 0
|
||||
if dropped:
|
||||
rows.append(
|
||||
_notice("log.sink.dropped", f"{dropped} log events were dropped", dropped=dropped)
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
try:
|
||||
# The connection is checked out BEFORE the guard, and every request path takes them in
|
||||
# that order too (its session already holds a connection when it reaches the guard). The
|
||||
# other way round, this thread would hold a process-global lock while waiting up to
|
||||
# `pool_timeout` for a connection, and the requests that could return one are the ones
|
||||
# blocked on the lock. The guard still spans the check, the insert and the commit.
|
||||
with self._bind.connect() as conn, logs.erasure_guard(), conn.begin():
|
||||
conn.execute(insert(LogEvent), without_erased(rows))
|
||||
except Exception as exc: # noqa: BLE001 — the database is down or refused the batch
|
||||
# silent-ok / noqa T201: the sink's OWN failure. A log call here would be handed to
|
||||
# this sink, which is the thing that just failed — straight into a loop. stderr is
|
||||
# the one way out that does not re-enter.
|
||||
print( # noqa: T201
|
||||
f"log sink: could not store {len(rows)} log events: "
|
||||
f"{logs.redact_text(str(exc))[:300]}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 0
|
||||
return len(rows)
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stopping.is_set():
|
||||
self.write_pending(wait=FLUSH_SECONDS)
|
||||
while self.write_pending(): # drain what arrived before the stop
|
||||
pass
|
||||
self.write_pending(final=True) # and the repeat notices of a window cut short
|
||||
|
||||
def start(self) -> None:
|
||||
self._writer = threading.Thread(target=self._run, name="log-sink", daemon=True)
|
||||
self._writer.start()
|
||||
|
||||
def stop(self, timeout: float = 5.0) -> None:
|
||||
self._stopping.set()
|
||||
if self._writer is not None:
|
||||
self._writer.join(timeout)
|
||||
|
||||
|
||||
_sink: LogSink | None = None
|
||||
|
||||
|
||||
def start() -> None:
|
||||
"""Attach the sink to the root logger and start its writer. Call after `logs.configure_logging`
|
||||
(which replaces the root handlers) and once the schema exists. A no-op when `LOG_DB_SINK` is off
|
||||
or it is already running."""
|
||||
global _sink
|
||||
if _sink is not None or not settings.log_db_sink:
|
||||
return
|
||||
_sink = LogSink()
|
||||
# The same level gate as stdout: the root may be lowered for one user's elevated requests, and
|
||||
# everyone else's detail must not reach the table through that gap either.
|
||||
_sink.addFilter(logs.LevelGate())
|
||||
logging.getLogger().addHandler(_sink)
|
||||
_sink.start()
|
||||
# A short-lived process (the download child) exits without a lifespan: flush on the way out.
|
||||
atexit.register(stop)
|
||||
|
||||
|
||||
def stop() -> None:
|
||||
"""Detach the sink and write what is still queued."""
|
||||
global _sink
|
||||
if _sink is None:
|
||||
return
|
||||
logging.getLogger().removeHandler(_sink)
|
||||
_sink.stop()
|
||||
_sink = None
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Stage the deletion of rows older than `retention_days` (at least one day — stored logs always
|
||||
expire). The caller commits. Returns the number of rows removed."""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(1, retention_days))
|
||||
result = db.execute(delete(LogEvent).where(LogEvent.ts < cutoff))
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def serialize(row: LogEvent) -> dict:
|
||||
"""A stored row as the API sends it — one shape for every reader: the admin console (E22.6) and
|
||||
a debug session's evidence list (E16)."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"ts": iso(row.ts),
|
||||
"level": row.level,
|
||||
"source": row.source,
|
||||
"logger": row.logger,
|
||||
"event": row.event,
|
||||
"msg": row.msg,
|
||||
"attrs": row.attrs,
|
||||
"exc": row.exc,
|
||||
"request_id": row.request_id,
|
||||
"user_id": row.user_id,
|
||||
"job_id": row.job_id,
|
||||
"debug_session_id": row.debug_session_id,
|
||||
"version": row.version,
|
||||
}
|
||||
@@ -0,0 +1,373 @@
|
||||
"""Bilingual text for every outbound mail, and the one resolver that decides which language a
|
||||
recipient gets (E9.5 S7).
|
||||
|
||||
Why a catalog instead of literals in `email.py`: HU/EN parity is a house rule for every user-facing
|
||||
string, and a mail is the one user-facing surface the reader CANNOT re-render by switching the UI
|
||||
language — it arrives once, in whatever language we picked, and stays that way. `test_mail_language.py`
|
||||
enforces that every key carries both languages *and* the same `{placeholders}`, so a half-translated
|
||||
mail fails the gate instead of reaching a person.
|
||||
|
||||
The language is resolved by the REQUEST handler and handed to `send_*` as a plain code. It is never
|
||||
resolved inside the mail: every send runs as a `BackgroundTasks` job, after the request's session is
|
||||
gone — the same reason `_welcome_mail_parts` computes its link in the handler and not in the task.
|
||||
"""
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Invite, User
|
||||
from app.utils import canonical_email
|
||||
|
||||
SUPPORTED: tuple[str, ...] = ("en", "hu")
|
||||
DEFAULT = "en"
|
||||
|
||||
|
||||
def normalize(lang: str | None) -> str:
|
||||
"""Coerce anything to a language we can actually render. Storage is not a guarantee: a
|
||||
`preferences.language` written by an older build (or a locale we no longer ship) must degrade to
|
||||
English rather than raise inside a background task nobody is watching."""
|
||||
return lang if lang in SUPPORTED else DEFAULT
|
||||
|
||||
|
||||
def text(key: str, lang: str | None, **values: object) -> str:
|
||||
"""Render one catalog entry. Unknown languages fall back to English (see `normalize`); an
|
||||
unknown KEY raises, because that is a programming error the tests must surface, not something to
|
||||
paper over with an empty line in someone's inbox."""
|
||||
return MAIL_TEXT[key][normalize(lang)].format(**values)
|
||||
|
||||
|
||||
def link_expiry(hours: int, lang: str) -> str:
|
||||
"""How long a link lasts, in a unit a reader converts on sight. A whole number of days is
|
||||
said in days — the approval mail's token lives seven of them, and "168 hours" is not an answer
|
||||
to "how long have I got". A single day stays "24 hours", which is how everyone writes it."""
|
||||
if hours >= 48 and hours % 24 == 0:
|
||||
return text("duration.days", lang, n=hours // 24)
|
||||
return text("duration.hours", lang, n=hours)
|
||||
|
||||
|
||||
def lang_for_user(user: User | None) -> str:
|
||||
"""The language a signed-up recipient gets: the one they picked in the app."""
|
||||
prefs = (user.preferences or {}) if user is not None else {}
|
||||
return normalize(prefs.get("language"))
|
||||
|
||||
|
||||
def lang_for_email(db: Session, email: str) -> str:
|
||||
"""The language for an address that may not have an account yet: their own account preference,
|
||||
else the language they filled our access-request form in, else English.
|
||||
|
||||
Matched CANONICALLY like every other identity lookup in this epic, and it walks past rows that
|
||||
carry no usable code instead of stopping at the first one — prod really does hold two accounts
|
||||
for a single mailbox, and only one of them may have ever picked a language.
|
||||
"""
|
||||
canon = canonical_email(email)
|
||||
for prefs in (
|
||||
db.execute(select(User.preferences).where(User.email_canonical == canon).order_by(User.id))
|
||||
.scalars()
|
||||
.all()
|
||||
):
|
||||
code = (prefs or {}).get("language")
|
||||
if code in SUPPORTED:
|
||||
return code
|
||||
for code in (
|
||||
db.execute(select(Invite.lang).where(Invite.email_canonical == canon).order_by(Invite.id))
|
||||
.scalars()
|
||||
.all()
|
||||
):
|
||||
if code in SUPPORTED:
|
||||
return code
|
||||
return DEFAULT
|
||||
|
||||
|
||||
# The brand line ("— Siftlode") and the product name are deliberately NOT catalog entries: they are
|
||||
# a name, not a translated string, and giving them keys would invite someone to "translate" them.
|
||||
MAIL_TEXT: dict[str, dict[str, str]] = {
|
||||
# ── How long a link lasts ───────────────────────────────────────────────────────────────
|
||||
# Rendered, not interpolated as a raw hour count: the approval mail's token lives seven days
|
||||
# (E9.5 S6), and "expires in 168 hours" is a unit no reader converts.
|
||||
"duration.hours": {"en": "{n} hours", "hu": "{n} óra"},
|
||||
"duration.days": {"en": "{n} days", "hu": "{n} nap"},
|
||||
# ── Shared closing fragments ────────────────────────────────────────────────────────────────
|
||||
"contact.reply": {
|
||||
"en": "\n\nQuestions? Just reply to this email ({admin}).",
|
||||
"hu": "\n\nKérdésed van? Csak válaszolj erre a levélre ({admin}).",
|
||||
},
|
||||
"contact.operator": {
|
||||
"en": "\n\nQuestions? Contact the operator at {operator}.",
|
||||
"hu": "\n\nKérdésed van? Írj az üzemeltetőnek: {operator}.",
|
||||
},
|
||||
"contact.deleted": {
|
||||
"en": "\n\nIf you didn't request this, contact the Siftlode operator at {operator}.",
|
||||
"hu": "\n\nHa nem te kérted, írj a Siftlode üzemeltetőjének: {operator}.",
|
||||
},
|
||||
"contact.suspended": {
|
||||
"en": (
|
||||
"If you think this is a mistake or have questions, contact the Siftlode operator "
|
||||
"at {operator}.\n\n"
|
||||
),
|
||||
"hu": (
|
||||
"Ha szerinted ez tévedés, vagy kérdésed van, írj a Siftlode üzemeltetőjének: "
|
||||
"{operator}.\n\n"
|
||||
),
|
||||
},
|
||||
"contact.suspended_unknown": {
|
||||
"en": "If you think this is a mistake, please contact the Siftlode operator.\n\n",
|
||||
"hu": "Ha szerinted ez tévedés, keresd a Siftlode üzemeltetőjét.\n\n",
|
||||
},
|
||||
# ── The one paragraph that carries the whole onboarding flow (see email._next_step) ──────────
|
||||
"next_step.verify": {
|
||||
"en": (
|
||||
"You already started an account, so there is just one step left. Confirm your email "
|
||||
"address and you're in:\n\n"
|
||||
"{link}\n\n"
|
||||
"The link works once and expires in {expiry}. If it does, you can ask for a fresh "
|
||||
"one from the sign-in page."
|
||||
),
|
||||
"hu": (
|
||||
"A fiókodat már elkezdted, úgyhogy egyetlen lépés maradt. Erősítsd meg az "
|
||||
"e-mail-címed, és bent is vagy:\n\n"
|
||||
"{link}\n\n"
|
||||
"A link egyszer használható, és {expiry} múlva lejár. Ha lejárt, a belépési oldalon "
|
||||
"kérhetsz újat."
|
||||
),
|
||||
},
|
||||
"next_step.sign_in": {
|
||||
"en": "Your account is ready — just sign in:\n\n{app}",
|
||||
"hu": "A fiókod készen áll — csak lépj be:\n\n{app}",
|
||||
},
|
||||
"next_step.register": {
|
||||
"en": (
|
||||
"Create your account whenever you feel like it:\n\n"
|
||||
"{app}\n\n"
|
||||
"We'll send you a confirmation link to finish, and then you're in."
|
||||
),
|
||||
"hu": (
|
||||
"Hozd létre a fiókod, amikor kedved tartja:\n\n"
|
||||
"{app}\n\n"
|
||||
"Küldünk egy megerősítő linket a befejezéshez, és már bent is vagy."
|
||||
),
|
||||
},
|
||||
# ── Access decision ─────────────────────────────────────────────────────────────────────────
|
||||
"access_approved.subject": {
|
||||
"en": "You're in — your Siftlode access is approved",
|
||||
"hu": "Bent vagy — jóváhagytuk a Siftlode-hozzáférésed",
|
||||
},
|
||||
"access_approved.body": {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Your request to use Siftlode has been approved — welcome aboard.\n\n"
|
||||
"{next_step}\n\n"
|
||||
"If you weren't expecting this, you can ignore the message.\n\n"
|
||||
"— Siftlode{contact}"
|
||||
),
|
||||
"hu": (
|
||||
"Szia,\n\n"
|
||||
"Jóváhagytuk a Siftlode-hozzáférési kérelmed — üdv a fedélzeten.\n\n"
|
||||
"{next_step}\n\n"
|
||||
"Ha nem számítottál erre, nyugodtan hagyd figyelmen kívül ezt a levelet.\n\n"
|
||||
"— Siftlode{contact}"
|
||||
),
|
||||
},
|
||||
"access_invitation.subject": {
|
||||
"en": "You're invited to Siftlode",
|
||||
"hu": "Meghívtak a Siftlode-ra",
|
||||
},
|
||||
"access_invitation.body": {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Someone has opened up Siftlode for you — a quiet, self-hosted reader for your YouTube "
|
||||
"subscriptions. Every upload from the channels you follow lands in one Feed you can "
|
||||
"actually filter and sort, with no algorithm deciding what you see, and no Shorts or "
|
||||
"livestreams unless you want them.\n\n"
|
||||
"{next_step}\n\n"
|
||||
"No rush — the door stays open.\n\n"
|
||||
"— Siftlode{contact}"
|
||||
),
|
||||
"hu": (
|
||||
"Szia,\n\n"
|
||||
"Valaki megnyitotta előtted a Siftlode-ot — ez egy csendes, saját üzemeltetésű olvasó "
|
||||
"a YouTube-feliratkozásaidhoz. A követett csatornák minden feltöltése egyetlen "
|
||||
"Feedbe érkezik, amit tényleg tudsz szűrni és rendezni; nincs algoritmus, ami "
|
||||
"eldönti, mit látsz, és nincs Shorts vagy élő adás, hacsak nem kéred.\n\n"
|
||||
"{next_step}\n\n"
|
||||
"Nem sürget semmi — az ajtó nyitva marad.\n\n"
|
||||
"— Siftlode{contact}"
|
||||
),
|
||||
},
|
||||
"access_denied.subject": {
|
||||
"en": "About your Siftlode access request",
|
||||
"hu": "A Siftlode-hozzáférési kérelmedről",
|
||||
},
|
||||
"access_denied.body": {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Thanks for your interest in Siftlode. After review, your access request wasn't "
|
||||
"approved this time.\n\n"
|
||||
"If you think this was a mistake, you're welcome to reply and let us know.\n\n"
|
||||
"— Siftlode{contact}"
|
||||
),
|
||||
"hu": (
|
||||
"Szia,\n\n"
|
||||
"Köszönjük az érdeklődésed a Siftlode iránt. Átnéztük a kérelmed, de most nem hagytuk "
|
||||
"jóvá.\n\n"
|
||||
"Ha szerinted ez tévedés, válaszolj nyugodtan erre a levélre.\n\n"
|
||||
"— Siftlode{contact}"
|
||||
),
|
||||
},
|
||||
# ── Account lifecycle ───────────────────────────────────────────────────────────────────────
|
||||
"role_changed.subject": {
|
||||
"en": "Your Siftlode role has changed",
|
||||
"hu": "Megváltozott a szereped a Siftlode-on",
|
||||
},
|
||||
"role_changed.granted": {
|
||||
"en": (
|
||||
"You've been granted admin access on Siftlode — you can now manage settings, the "
|
||||
"scheduler and users."
|
||||
),
|
||||
"hu": (
|
||||
"Admin jogot kaptál a Siftlode-on — mostantól kezelheted a beállításokat, az ütemezőt "
|
||||
"és a felhasználókat."
|
||||
),
|
||||
},
|
||||
"role_changed.revoked": {
|
||||
"en": "Your admin access on Siftlode has been removed — you're now a regular user.",
|
||||
"hu": "Elvettük az admin jogod a Siftlode-on — mostantól sima felhasználó vagy.",
|
||||
},
|
||||
"role_changed.body": {
|
||||
"en": "Hi,\n\n{line}{contact}\n\n— Siftlode",
|
||||
"hu": "Szia,\n\n{line}{contact}\n\n— Siftlode",
|
||||
},
|
||||
"account_unsuspended.subject": {
|
||||
"en": "Your Siftlode account has been reinstated",
|
||||
"hu": "Visszaállítottuk a Siftlode-fiókod",
|
||||
},
|
||||
"account_unsuspended.body": {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Your Siftlode account has been reinstated — the suspension was lifted and you can "
|
||||
"sign in again.{contact}\n\n— Siftlode"
|
||||
),
|
||||
"hu": (
|
||||
"Szia,\n\n"
|
||||
"Visszaállítottuk a Siftlode-fiókod — a felfüggesztést feloldottuk, újra be tudsz "
|
||||
"lépni.{contact}\n\n— Siftlode"
|
||||
),
|
||||
},
|
||||
"account_deleted.subject": {
|
||||
"en": "Your Siftlode account has been deleted",
|
||||
"hu": "Töröltük a Siftlode-fiókod",
|
||||
},
|
||||
"account_deleted.body": {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Your Siftlode account and all associated data — subscriptions, tags, watch history, "
|
||||
"playlists and settings — have been permanently deleted. Nothing is retained.\n\n"
|
||||
"Thanks for having used Siftlode.{contact}\n\n— Siftlode"
|
||||
),
|
||||
"hu": (
|
||||
"Szia,\n\n"
|
||||
"A Siftlode-fiókod és minden hozzá tartozó adat — feliratkozások, címkék, megtekintési "
|
||||
"előzmények, lejátszási listák és beállítások — véglegesen törölve. Semmit nem őrzünk "
|
||||
"meg.\n\n"
|
||||
"Köszönjük, hogy használtad a Siftlode-ot.{contact}\n\n— Siftlode"
|
||||
),
|
||||
},
|
||||
"account_suspended.subject": {
|
||||
"en": "Your Siftlode account is suspended",
|
||||
"hu": "A Siftlode-fiókod fel van függesztve",
|
||||
},
|
||||
"account_suspended.body": {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"A sign-in to your Siftlode account was just attempted, but the account is currently "
|
||||
"suspended, so access was denied.\n\n{contact}— Siftlode"
|
||||
),
|
||||
"hu": (
|
||||
"Szia,\n\n"
|
||||
"Valaki most próbált belépni a Siftlode-fiókodba, de a fiók jelenleg fel van "
|
||||
"függesztve, ezért a belépést elutasítottuk.\n\n{contact}— Siftlode"
|
||||
),
|
||||
},
|
||||
# ── To the admins ───────────────────────────────────────────────────────────────────────────
|
||||
"admin_new_request.subject": {
|
||||
"en": "Siftlode access request from {requester}",
|
||||
"hu": "Siftlode hozzáférési kérelem: {requester}",
|
||||
},
|
||||
"admin_new_request.body": {
|
||||
"en": (
|
||||
"{requester} just requested access to Siftlode.\n\n"
|
||||
"Approve or deny it here:\n{link}\n"
|
||||
"(or in Siftlode: open Users → Access requests)\n\n"
|
||||
"Requester's email: {requester}\n"
|
||||
"Replying to this message goes straight to the requester (it's the Reply-To address).\n"
|
||||
),
|
||||
"hu": (
|
||||
"{requester} most kért hozzáférést a Siftlode-hoz.\n\n"
|
||||
"Itt hagyhatod jóvá vagy utasíthatod el:\n{link}\n"
|
||||
"(vagy a Siftlode-ban: Felhasználók → Hozzáférési kérések)\n\n"
|
||||
"A kérelmező e-mail-címe: {requester}\n"
|
||||
"Ha erre a levélre válaszolsz, az egyenesen a kérelmezőhöz megy (ő a Reply-To cím).\n"
|
||||
),
|
||||
},
|
||||
# ── Link mails ──────────────────────────────────────────────────────────────────────────────
|
||||
"verify_email.subject": {
|
||||
"en": "Confirm your Siftlode email",
|
||||
"hu": "Erősítsd meg a Siftlode e-mail-címed",
|
||||
},
|
||||
"verify_email.body": {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Confirm your email to finish creating your Siftlode account:\n\n"
|
||||
"{link}\n\n"
|
||||
"The link works once and expires in {expiry}. If it does, you can ask for a fresh "
|
||||
"one from the sign-in page.\n\n"
|
||||
"If you didn't sign up, you can ignore this message.\n\n"
|
||||
"— Siftlode"
|
||||
),
|
||||
"hu": (
|
||||
"Szia,\n\n"
|
||||
"Erősítsd meg az e-mail-címed, hogy elkészüljön a Siftlode-fiókod:\n\n"
|
||||
"{link}\n\n"
|
||||
"A link egyszer használható, és {expiry} múlva lejár. Ha lejárt, a belépési oldalon "
|
||||
"kérhetsz újat.\n\n"
|
||||
"Ha nem te regisztráltál, nyugodtan hagyd figyelmen kívül ezt a levelet.\n\n"
|
||||
"— Siftlode"
|
||||
),
|
||||
},
|
||||
"password_reset.subject": {
|
||||
"en": "Reset your Siftlode password",
|
||||
"hu": "Állíts be új Siftlode-jelszót",
|
||||
},
|
||||
"password_reset.body": {
|
||||
"en": (
|
||||
"Hi,\n\n"
|
||||
"Use this link to set a new Siftlode password (it expires in an hour):\n\n"
|
||||
"{link}\n\n"
|
||||
"If you didn't request this, you can ignore this message — your password is "
|
||||
"unchanged.\n\n"
|
||||
"— Siftlode"
|
||||
),
|
||||
"hu": (
|
||||
"Szia,\n\n"
|
||||
"Ezzel a linkkel állíthatsz be új Siftlode-jelszót (egy óra múlva lejár):\n\n"
|
||||
"{link}\n\n"
|
||||
"Ha nem te kérted, hagyd figyelmen kívül ezt a levelet — a jelszavad változatlan.\n\n"
|
||||
"— Siftlode"
|
||||
),
|
||||
},
|
||||
"test.subject": {
|
||||
"en": "Siftlode SMTP test",
|
||||
"hu": "Siftlode SMTP-teszt",
|
||||
},
|
||||
"test.body": {
|
||||
"en": (
|
||||
"This is a test email from Siftlode.\n\n"
|
||||
"If you're reading this, your SMTP settings work.\n\n"
|
||||
"— Siftlode"
|
||||
),
|
||||
"hu": (
|
||||
"Ez egy teszt e-mail a Siftlode-tól.\n\n"
|
||||
"Ha ezt olvasod, az SMTP-beállításaid működnek.\n\n"
|
||||
"— Siftlode"
|
||||
),
|
||||
},
|
||||
}
|
||||
+105
-18
@@ -1,38 +1,45 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import math
|
||||
from contextlib import asynccontextmanager, suppress
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
|
||||
# When started via uvicorn --log-config the "siftlode" logger is already configured
|
||||
# (see log_config.json). This block is a timestamped fallback for other entrypoints
|
||||
# (tests, scripts) so our logs are never silently dropped.
|
||||
_siftlode_logger = logging.getLogger("siftlode")
|
||||
if not _siftlode_logger.handlers:
|
||||
_handler = logging.StreamHandler(sys.stdout)
|
||||
_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s [%(name)s] %(message)s"))
|
||||
_siftlode_logger.addHandler(_handler)
|
||||
_siftlode_logger.setLevel(logging.INFO)
|
||||
_siftlode_logger.propagate = False
|
||||
# Logging is configured HERE, before the route modules are imported, so every logger they create at
|
||||
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
||||
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
||||
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
||||
from app import logs
|
||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||
|
||||
log = logging.getLogger("siftlode.app")
|
||||
configure_logging("api")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy.exc import DataError
|
||||
from starlette.middleware.sessions import SessionMiddleware
|
||||
|
||||
from app import logsink
|
||||
from app import state
|
||||
from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.plex import stream as plex_stream
|
||||
from app.plex import subtitles as plex_subtitles
|
||||
from app.routes import (
|
||||
admin,
|
||||
admin_logs as admin_logs_routes,
|
||||
audit as audit_routes,
|
||||
auth as auth_routes,
|
||||
channels,
|
||||
config as config_routes,
|
||||
debug_sessions as debug_sessions_routes,
|
||||
diagnostics as diagnostics_routes,
|
||||
discover as discover_routes,
|
||||
downloads,
|
||||
feed,
|
||||
health,
|
||||
@@ -57,16 +64,28 @@ from app.scheduler import shutdown_scheduler, start_scheduler
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
log.info("Siftlode starting up")
|
||||
# The stored copy of the log (log_events). The entrypoint has already migrated the schema.
|
||||
logsink.start()
|
||||
# First-run: if the instance isn't configured yet, mint a fresh one-time setup token and print
|
||||
# the wizard URL so the operator can open it (the app runs in setup mode until they finish).
|
||||
with SessionLocal() as db:
|
||||
# The admin's runtime level (sysconfig `log_level`) wins over the LOG_LEVEL bootstrap
|
||||
# default, so a level chosen in the UI survives a restart without touching the environment.
|
||||
# Best-effort on purpose: a LOGGING knob must never be the thing that stops the app from
|
||||
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
||||
try:
|
||||
set_level(sysconfig.get_str(db, "log_level"))
|
||||
except Exception: # noqa: BLE001 — any DB/config failure keeps the bootstrap level
|
||||
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
||||
# and `%s` of the exception does not say which of them raised.
|
||||
log.exception("could not read the stored log level; staying at %s", current_level())
|
||||
if not state.is_configured(db):
|
||||
token = state.rotate_setup_token(db)
|
||||
url = f"{settings.app_base}/setup?token={token}"
|
||||
log.warning(
|
||||
"FIRST-RUN SETUP REQUIRED — this instance isn't configured yet.\n"
|
||||
" Open the install wizard (internal access only):\n %s",
|
||||
url,
|
||||
url, # log-pii-ok: this instance's OWN setup URL, and `?token=` is redacted
|
||||
)
|
||||
# No HLS session survives a restart (they live in this process), so anything still on disk is
|
||||
# a crashed run's segments — sweep them or they accumulate forever. FIRE AND FORGET, not
|
||||
@@ -76,6 +95,7 @@ async def lifespan(app: FastAPI):
|
||||
# alone doesn't buy that; not waiting does. The sweep only touches directories no live session
|
||||
# owns, so racing it against early playback is safe.
|
||||
sweep = asyncio.create_task(_sweep_hls_orphans())
|
||||
reaper = asyncio.create_task(_reap_hls_sessions())
|
||||
start_scheduler()
|
||||
try:
|
||||
yield
|
||||
@@ -85,10 +105,37 @@ async def lifespan(app: FastAPI):
|
||||
# pending ("Task was destroyed but it is pending!" — noise that reads like a bug during
|
||||
# triage). Cancelling does NOT interrupt the rmtree thread itself; this only makes our own
|
||||
# bookkeeping honest.
|
||||
sweep.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await sweep
|
||||
for task in (sweep, reaper):
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task
|
||||
shutdown_scheduler()
|
||||
# Kills a running subtitle extraction: its worker would otherwise hold interpreter exit for
|
||||
# the rest of the read (see plex_subtitles.shutdown).
|
||||
plex_subtitles.shutdown()
|
||||
logsink.stop() # last: write what the shutdown itself just logged
|
||||
|
||||
|
||||
# How often idle HLS sessions are reaped (see plex/stream.py `reap_idle`).
|
||||
_HLS_REAP_EVERY_S = 60.0
|
||||
|
||||
|
||||
async def _reap_hls_sessions() -> None:
|
||||
"""The periodic reaper plex/stream.py's docstring promises. `reap_idle` used to run only inside
|
||||
the NEXT session request, so after the last playback of the day an abandoned session — its
|
||||
ffmpeg still working through the film, its directory a whole segmented copy of it — lived until
|
||||
somebody pressed play again or the process restarted.
|
||||
|
||||
A FINISHED session is reaped on the short rule here, not kept for the full idle window: an open
|
||||
player checkpoints every 10 s and that route touches the session, so a paused viewer is no longer
|
||||
invisible to this pass (see `reap_idle`). Keeping every finished session for ten minutes instead
|
||||
held one whole segmented copy of a film per item on the download volume (round 5)."""
|
||||
while True:
|
||||
await asyncio.sleep(_HLS_REAP_EVERY_S)
|
||||
try:
|
||||
await asyncio.to_thread(plex_stream.reap_idle)
|
||||
except Exception:
|
||||
log.exception("HLS session reap failed")
|
||||
|
||||
|
||||
async def _sweep_hls_orphans() -> None:
|
||||
@@ -107,6 +154,34 @@ async def _sweep_hls_orphans() -> None:
|
||||
|
||||
app = FastAPI(title=settings.app_name, lifespan=lifespan)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def _validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
"""FastAPI's own 422, in its own shape — except that it can be ENCODED. The JSON parser accepts
|
||||
`NaN`/`Infinity`, a float field refuses them, and the refusal echoes the value back: the default
|
||||
handler then failed to serialise its own error (JSON has no NaN) and the request became a 500
|
||||
with a traceback. A non-finite value is echoed as its name instead."""
|
||||
finite = {float: lambda x: x if math.isfinite(x) else str(x)}
|
||||
return JSONResponse(
|
||||
status_code=422, content={"detail": jsonable_encoder(exc.errors(), custom_encoder=finite)}
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(DataError)
|
||||
async def _nul_in_text(request: Request, exc: DataError) -> JSONResponse:
|
||||
"""A NUL byte in a string the request sent is a 422, on every route at once. Postgres cannot hold
|
||||
NUL in text, so the driver refuses to even send the query — and a `%00` in a query parameter or
|
||||
an escaped NUL in a JSON body (a playlist's `contains=`, a reorder's key list, a collection
|
||||
search) was a 500 with a traceback wherever it reached a comparison. Guarding each site did not
|
||||
hold: round 3 guarded the by-rating_key lookup and missed the rest. Any other DataError is left
|
||||
alone — re-raised, it stays the 500 it always was."""
|
||||
if "NUL (0x00)" not in str(exc.orig):
|
||||
raise exc
|
||||
return JSONResponse(
|
||||
status_code=422, content={"detail": {"code": "invalid_input", "reason": "nul"}}
|
||||
)
|
||||
|
||||
|
||||
app.add_middleware(
|
||||
SessionMiddleware,
|
||||
secret_key=settings.secret_key,
|
||||
@@ -182,13 +257,24 @@ async def security_headers(request, call_next):
|
||||
return response
|
||||
|
||||
|
||||
# The access log (app/logs.py): request id, route template, status, duration — one line, never a
|
||||
# query string. Registered LAST so it is the OUTERMOST http middleware and therefore times the whole
|
||||
# request, including what the three above do. Being outermost also puts it OUTSIDE SessionMiddleware,
|
||||
# which is why the account comes from `request.state.user_id` (set by the auth dependency, the one
|
||||
# place a session is actually resolved into an account) rather than from `request.session`.
|
||||
app.middleware("http")(log_requests)
|
||||
|
||||
|
||||
app.include_router(health.router)
|
||||
app.include_router(auth_routes.router)
|
||||
app.include_router(sync.router)
|
||||
app.include_router(tags.router)
|
||||
app.include_router(feed.router)
|
||||
app.include_router(discover_routes.router)
|
||||
app.include_router(search_routes.router)
|
||||
app.include_router(me.router)
|
||||
app.include_router(diagnostics_routes.router)
|
||||
app.include_router(debug_sessions_routes.router)
|
||||
app.include_router(notifications.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(channels.router)
|
||||
@@ -202,6 +288,7 @@ app.include_router(admin.router)
|
||||
app.include_router(config_routes.router)
|
||||
app.include_router(scheduler_routes.router)
|
||||
app.include_router(audit_routes.router)
|
||||
app.include_router(admin_logs_routes.router)
|
||||
app.include_router(quota.router)
|
||||
app.include_router(version.router)
|
||||
app.include_router(setup_routes.router)
|
||||
|
||||
+264
-3
@@ -3,6 +3,7 @@ from datetime import date, datetime
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
Computed,
|
||||
Date,
|
||||
DateTime,
|
||||
@@ -11,15 +12,23 @@ from sqlalchemy import (
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
LargeBinary,
|
||||
SmallInteger,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
|
||||
from app.db import Base
|
||||
from app.utils import canonical_email
|
||||
|
||||
|
||||
# The resume-position columns (VideoState / PlexState.position_seconds) are int4. The checkpoint
|
||||
# routes bound their bodies by this, so an out-of-range position is a 422 rather than a 500.
|
||||
POSITION_MAX_S = 2**31 - 1
|
||||
|
||||
|
||||
class TimestampMixin:
|
||||
@@ -29,6 +38,21 @@ class TimestampMixin:
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
|
||||
class CanonicalEmailMixin:
|
||||
"""Keeps `email_canonical` derived from `email` on every write.
|
||||
|
||||
The invariant — "identity is matched on the canonical form, mail is sent to the typed one" —
|
||||
lives HERE, in code, instead of in a comment every write site has to remember. That distinction
|
||||
is what the 2026-09-03 lockout was made of: one gate lowercased, nothing else did, and an
|
||||
approved user's Google sign-in silently became a second access request.
|
||||
"""
|
||||
|
||||
@validates("email")
|
||||
def _derive_email_canonical(self, _key: str, value: str | None) -> str | None:
|
||||
self.email_canonical = canonical_email(value)
|
||||
return value
|
||||
|
||||
|
||||
class UpdatedAtMixin:
|
||||
"""Adds an `updated_at` that server-defaults on insert and bumps on every UPDATE."""
|
||||
|
||||
@@ -37,7 +61,7 @@ class UpdatedAtMixin:
|
||||
)
|
||||
|
||||
|
||||
class User(Base, TimestampMixin):
|
||||
class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
__tablename__ = "users"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
@@ -45,8 +69,26 @@ class User(Base, TimestampMixin):
|
||||
# multiple NULLs under a unique index). Set on Google sign-in / in-app linking.
|
||||
google_sub: Mapped[str | None] = mapped_column(String(64), unique=True, index=True)
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
||||
# The same mailbox, in the form used to MATCH identities (utils.canonical_email) — Gmail dots and
|
||||
# `+tags` folded away. Kept by the validator below so no write site can forget it. Deliberately
|
||||
# NOT unique: prod already carries two rows for one mailbox (a password registration and the
|
||||
# Google sign-in that failed to find it), and a migration must not merge accounts on its own.
|
||||
email_canonical: Mapped[str] = mapped_column(String(320), index=True, server_default="")
|
||||
# The address Google asserted for this account, recorded at sign-in / link time. NOT the same
|
||||
# thing as `email`: since E9.5 S3 a Google sign-in no longer overwrites the address the user
|
||||
# registered with, so a deliberate `me+siftlode@gmail.com` survives — and this is where the other
|
||||
# half of the truth lives, for the UI to show when the two differ. NULL until the next Google
|
||||
# sign-in for accounts linked before this column existed.
|
||||
google_email: Mapped[str | None] = mapped_column(String(320))
|
||||
# argon2id hash for email+password login; NULL for Google-only / demo accounts.
|
||||
password_hash: Mapped[str | None] = mapped_column(String(255))
|
||||
# When SEC-10 dropped a preset password on Google adoption (see the single caller of
|
||||
# auth.revoke_preset_password_if_unverified; admin approval no longer revokes — E9.5 S1). Without
|
||||
# this trace a revoked account was indistinguishable from a Google-only one, so it failed the
|
||||
# password-reset eligibility check and could never be recovered — it could neither sign in nor
|
||||
# reset. Set = "had a password, lost it to the guard" -> reset-eligible. Cleared whenever a
|
||||
# password is set again (reset-confirm / set-password).
|
||||
password_revoked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Email ownership proven (Google sign-in is implicitly verified; password registration
|
||||
# verifies via a link). Login requires this AND is_active.
|
||||
email_verified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
@@ -62,6 +104,16 @@ class User(Base, TimestampMixin):
|
||||
# current_user rejects any cookie whose epoch is stale, so a stolen/copied cookie dies on those
|
||||
# events (client-side signed cookies otherwise stay valid until they expire).
|
||||
session_epoch: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# When this account last signed in (Google or password), stamped by the one sign-in path,
|
||||
# routes/auth._establish_session. NULL = not since the column existed (0085; no backfill is
|
||||
# possible). The shared demo account is never stamped — its "last login" would mean nobody.
|
||||
last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The diagnostics level this user consented to (E22 S5) and when it lapses back to the base
|
||||
# level. Both NULL = base. Written only by `PUT /api/me/diagnostics`, which computes the expiry
|
||||
# itself; read through `app.diagnostics.effective_level`, never directly (an expired level is
|
||||
# not cleared eagerly — it simply stops counting).
|
||||
diagnostics_level: Mapped[str | None] = mapped_column(String(8))
|
||||
diagnostics_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
display_name: Mapped[str | None] = mapped_column(String(255))
|
||||
avatar_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
role: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
@@ -114,7 +166,7 @@ class AuthToken(Base, TimestampMixin):
|
||||
used_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class Invite(Base):
|
||||
class Invite(Base, CanonicalEmailMixin):
|
||||
"""Access-request / whitelist row. Source of truth for who may sign in; the env
|
||||
ALLOWED_EMAILS/ADMIN_EMAILS act only as a bootstrap fallback (see auth.is_allowed)."""
|
||||
|
||||
@@ -122,6 +174,9 @@ class Invite(Base):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
||||
# See User.email_canonical. Not unique here either: prod has approved invites for both spellings
|
||||
# of the same mailbox, dating back to 2026-06-16.
|
||||
email_canonical: Mapped[str] = mapped_column(String(320), index=True, server_default="")
|
||||
# pending | approved | denied
|
||||
status: Mapped[str] = mapped_column(String(16), default="pending", server_default="pending")
|
||||
note: Mapped[str | None] = mapped_column(Text)
|
||||
@@ -130,6 +185,17 @@ class Invite(Base):
|
||||
)
|
||||
decided_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
decided_by: Mapped[str | None] = mapped_column(String(320)) # admin email
|
||||
# When the admin last SENT this person an invitation from the whitelist row. Adding an address
|
||||
# stays silent — it opens a door, it does not knock — so telling them is a separate, deliberate
|
||||
# act, and this is what stops it happening twice to a real person.
|
||||
invited_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The language this person asked us in ("en"/"hu"), captured when the request was made: the page
|
||||
# their form was on, or the locale Google reported for a refused sign-in. NULL for a row the
|
||||
# admin whitelisted by hand — nobody has told us anything about that mailbox, which is exactly
|
||||
# why the invitation send asks (E9.5 S7). Captured rather than derived for the same reason
|
||||
# `preferences.language` and `google_email` are: the signal exists only in that one moment and
|
||||
# cannot be backfilled afterwards.
|
||||
lang: Mapped[str | None] = mapped_column(String(8))
|
||||
|
||||
|
||||
class DemoWhitelist(Base, TimestampMixin):
|
||||
@@ -176,6 +242,9 @@ class Channel(Base, TimestampMixin):
|
||||
details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# When the newest uploads were last read through the Data API because the RSS feed would not
|
||||
# answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock.
|
||||
feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
||||
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this channel only entered the catalog because a live YouTube search
|
||||
@@ -374,6 +443,7 @@ class VideoState(Base, UpdatedAtMixin):
|
||||
# status: a partially-watched video keeps status "new" but a non-zero position, so
|
||||
# it can render a progress bar and match the "in progress" feed filter. 0 = not
|
||||
# started / finished (trivially-early and near-finished positions are not stored).
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(
|
||||
Integer, default=0, server_default="0", nullable=False
|
||||
)
|
||||
@@ -484,6 +554,91 @@ class AuditLog(Base):
|
||||
)
|
||||
|
||||
|
||||
class LogEvent(Base):
|
||||
"""One stored log event (E22 S3) — the `log_events` table the diagnostics console reads. Rows are
|
||||
written by app/logsink.py's background writer, never inside a request's transaction, already
|
||||
redacted (the same `logs.record_fields` the stdout line is built from).
|
||||
|
||||
`user_id` has NO foreign key on purpose: the writer runs behind the event, so a line about an
|
||||
account can arrive after that account was purged, and a constraint would make it fail the whole
|
||||
batch it travels in. `auth.purge_user` deletes an account's rows explicitly instead, after
|
||||
`logs.forget_user` makes every later write in this process store such a line without the account;
|
||||
the retention job (`log_retention_days`) bounds whatever another process writes late."""
|
||||
|
||||
__tablename__ = "log_events"
|
||||
__table_args__ = (
|
||||
Index("ix_log_events_user_ts", "user_id", "ts"),
|
||||
# E16: an issue's bundle is `WHERE debug_session_id = …`; nearly every row has none.
|
||||
Index(
|
||||
"ix_log_events_debug_session",
|
||||
"debug_session_id",
|
||||
postgresql_where="debug_session_id IS NOT NULL",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True)
|
||||
ts: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
level: Mapped[str] = mapped_column(String(10))
|
||||
sev: Mapped[int] = mapped_column(SmallInteger)
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
logger: Mapped[str] = mapped_column(String(128))
|
||||
event: Mapped[str | None] = mapped_column(String(64))
|
||||
msg: Mapped[str] = mapped_column(Text)
|
||||
attrs: Mapped[dict | None] = mapped_column(JSONB)
|
||||
exc: Mapped[str | None] = mapped_column(Text)
|
||||
request_id: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
user_id: Mapped[int | None] = mapped_column(Integer)
|
||||
debug_session_id: Mapped[str | None] = mapped_column(String(64))
|
||||
job_id: Mapped[int | None] = mapped_column(Integer)
|
||||
version: Mapped[str | None] = mapped_column(String(64))
|
||||
|
||||
|
||||
class DebugSession(Base):
|
||||
"""One recording: Start → reproduce → Stop (E16, app/debugsession.py). While it is open, every
|
||||
request of its user that names it (`X-Debug-Session`) is logged at `level` and tagged with `id`,
|
||||
and the issue it ends in is built from the rows carrying that tag. `until` is the hard cap the
|
||||
server set at Start; an open session past it is over whether or not the sweep has closed it yet.
|
||||
One OPEN session per user (`uq_debug_sessions_open_per_user`, partial on `ended_at IS NULL`)."""
|
||||
|
||||
__tablename__ = "debug_sessions"
|
||||
__table_args__ = (
|
||||
Index(
|
||||
"uq_debug_sessions_open_per_user",
|
||||
"user_id",
|
||||
unique=True,
|
||||
postgresql_where="ended_at IS NULL",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
level: Mapped[str] = mapped_column(String(8))
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
until: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
ended_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# app/debugsession.py `EndReason`: stopped by the user, or past `until`.
|
||||
ended_reason: Mapped[str | None] = mapped_column(String(16))
|
||||
issue_url: Mapped[str | None] = mapped_column(String(512))
|
||||
# The issue being written from it (E16 S3; app/debugsession.py `save_draft` holds its shape).
|
||||
draft: Mapped[dict | None] = mapped_column(JSONB)
|
||||
|
||||
|
||||
class DebugScreenshot(Base):
|
||||
"""A screenshot attached to a debug session (E16) — pasted, dropped, or captured from the pill.
|
||||
Stored in the row: bounded per session (app/debugsession.py), and gone with it by cascade."""
|
||||
|
||||
__tablename__ = "debug_screenshots"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
session_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("debug_sessions.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
mime: Mapped[str] = mapped_column(String(16))
|
||||
size: Mapped[int] = mapped_column(Integer)
|
||||
data: Mapped[bytes] = mapped_column(LargeBinary)
|
||||
|
||||
|
||||
class AppState(Base):
|
||||
"""Single-row global app state (admin-controlled)."""
|
||||
|
||||
@@ -551,6 +706,10 @@ class Playlist(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# SHA-256 of name + ordered video ids as last synced from / pushed to YouTube; the
|
||||
# baseline `dirty` is derived against. NULL = no baseline yet (treated as dirty).
|
||||
synced_fingerprint: Mapped[str | None] = mapped_column(String)
|
||||
# JSON list of the ordered video ids as of the last sync — the 3-way-merge baseline the
|
||||
# two-way YouTube sync classifies each item against (local-add vs YT-add vs real delete).
|
||||
# NULL = no baseline yet; the first merge then does a lossless union (never deletes).
|
||||
synced_video_ids: Mapped[str | None] = mapped_column(Text)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
items: Mapped[list["PlaylistItem"]] = relationship(
|
||||
@@ -704,6 +863,12 @@ class MediaAsset(Base, TimestampMixin):
|
||||
container: Mapped[str | None] = mapped_column(String(16))
|
||||
vcodec: Mapped[str | None] = mapped_column(String(32))
|
||||
acodec: Mapped[str | None] = mapped_column(String(32))
|
||||
# "video" | "audio" | "file" — the media-kind discriminator that drives which library actions
|
||||
# apply (play/edit for media; download-only for a generic uploaded file). Null while an asset is
|
||||
# still pending (kind is known once the file is probed at finalize). It is the single stored home
|
||||
# of the invariant `downloads.mediakind.derive` used to compute — never re-derived in prose (an
|
||||
# earlier ad-hoc `vcodec is None and acodec` check lived only in links.py; consolidated here).
|
||||
media_kind: Mapped[str | None] = mapped_column(String(16))
|
||||
nfo_written: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Set once the GC has warned holders of the upcoming expiry, so it doesn't warn every run.
|
||||
gc_notified: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
@@ -806,6 +971,8 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
error_code: Mapped[str | None] = mapped_column(
|
||||
String(24)
|
||||
) # category, see downloads.errors.classify
|
||||
# The host-specific failure-catalog entry on top of the category (`facebook.login_wall`), or NULL.
|
||||
error_detail: Mapped[str | None] = mapped_column(String(48)) # see downloads.errors.diagnose
|
||||
queue_pos: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
@@ -867,6 +1034,35 @@ class DownloadLink(Base, TimestampMixin):
|
||||
view_count: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
|
||||
|
||||
class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's stored login cookies for ONE auth-walled download host (Facebook, Instagram, …).
|
||||
|
||||
Per-user, per-host: each user pastes their OWN browser cookies.txt (Netscape format) for a host
|
||||
where they're logged in (e.g. a closed Facebook group they belong to), and a download of a
|
||||
yt-dlp-supported, auth-walled link on that host is retried with the ACTING user's cookie. The
|
||||
blob is Fernet-encrypted at rest (`cookies_enc`) and is NEVER returned by any API/log/audit —
|
||||
the routes are write-only (host + updated_at only). See `downloads.credentials`.
|
||||
|
||||
Two load-bearing security controls live around this row, NOT on it:
|
||||
* **cache privacy** — a cookie-authed download SALTS its MediaAsset `format_sig` with the
|
||||
user id, so the authed asset is private to that user and can never be cache-served to
|
||||
another user (`downloads.credentials.cookie_salt`).
|
||||
* **host scope (SEC-12)** — a cookie is attached ONLY when the download URL's netloc matches
|
||||
this row's `host_key` (`downloads.credentials.host_key_for_url`), so an arbitrary `?url=`
|
||||
can't exfiltrate the user's session to a look-alike host.
|
||||
|
||||
`host_key` is a short canonical key from the supported-host registry (e.g. "facebook"), not the
|
||||
raw netloc — one credential can cover several domains of the same site (facebook.com/fb.watch)."""
|
||||
|
||||
__tablename__ = "download_credentials"
|
||||
__table_args__ = (UniqueConstraint("user_id", "host_key", name="uq_download_credential"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
host_key: Mapped[str] = mapped_column(String(64))
|
||||
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
|
||||
|
||||
class MessageKey(Base, TimestampMixin):
|
||||
"""A user's end-to-end-encryption key material for direct messaging (phase 2).
|
||||
|
||||
@@ -1088,6 +1284,22 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# Browser-playability class from ffprobe: "direct" | "remux" | "transcode" (None = unprobed).
|
||||
playable: Mapped[str | None] = mapped_column(String(12), index=True)
|
||||
probed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Transcode HW-capability facet (E7 playback-class filter). NULL for non-transcode items; for a
|
||||
# `playable="transcode"` item: True = the prod iGPU can hardware-transcode it (codec is HW-decodable
|
||||
# AND the source is 8-bit SDR), False = software libx264 only (AV1/XviD, or a 10-bit/HDR source that
|
||||
# breaks the VAAPI nv12 pipeline — see plex/stream.py). Derived by `plex.sync.transcode_hw_capable`:
|
||||
# set codec-only at sync (bit depth unknown from the cheap listing), then refined by the bit-depth
|
||||
# backfill (`_probe_media_batch`) which downgrades a 10-bit/HDR HEVC to False. The one filterable
|
||||
# home of the invariant `_want_hw`'s source gate computes at play time — kept in sync via the shared
|
||||
# classifier so the catalog facet and the runtime encoder choice never disagree by codec.
|
||||
transcode_hw: Mapped[bool | None] = mapped_column(Boolean, index=True)
|
||||
# Video bit depth (8/10/12…) from the Plex media Stream, filled by the bit-depth backfill. NULL =
|
||||
# not yet probed. Surfaced on the info page (e.g. "HEVC · 10-bit") and feeds `transcode_hw`.
|
||||
video_bit_depth: Mapped[int | None] = mapped_column(Integer)
|
||||
# Watermark for the bit-depth backfill: when this item's media Stream was last probed for
|
||||
# bit-depth/HDR. NULL = never (the backfill's work-list). Only HW-decodable transcode items are
|
||||
# probed (the rest are SW-only by codec and need no bit-depth); see `_probe_media_batch`.
|
||||
media_probed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Intro/credit markers: [{"type": "intro"|"credits", "start_s": int, "end_s": int}, ...]
|
||||
markers: Mapped[list | None] = mapped_column(JSON)
|
||||
added_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
@@ -1303,3 +1515,52 @@ class PlexPlaylistItem(Base):
|
||||
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||
|
||||
|
||||
class ComfortPin(Base, TimestampMixin):
|
||||
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X" (E24).
|
||||
|
||||
Deliberately NOT a boolean on `VideoState`/`PlexState`, and NOT a polymorphic `(kind, ref)` string
|
||||
pair. A flag cannot ORDER a shelf and cannot hold a channel or a show — and comfort media is
|
||||
usually "put on X", a series or a channel, not one particular video. A string ref would order and
|
||||
hold anything, but it would also lose the foreign key: a Plex item dropped from the library, or an
|
||||
unsubscribed channel, would leave a dead tile on the one surface whose whole job is to be
|
||||
reliable. So instead: one nullable CASCADE-ing FK per pinnable kind, and a check constraint that
|
||||
exactly one of them is set.
|
||||
|
||||
That constraint is the house rule applied to a five-way union — the invariant lives in the schema,
|
||||
not in a comment every insert site has to remember. `kind`/`ref` below derive FROM the columns, so
|
||||
there is no second vocabulary to keep in sync.
|
||||
|
||||
NULLs are DISTINCT in a Postgres unique index, so the five per-kind unique constraints each stop a
|
||||
double-pin of the same thing without colliding across kinds."""
|
||||
|
||||
__tablename__ = "comfort_pins"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"num_nonnulls(video_id, channel_id, plex_item_id, plex_show_id, playlist_id) = 1",
|
||||
name="ck_comfort_pin_one_ref",
|
||||
),
|
||||
UniqueConstraint("user_id", "video_id", name="uq_comfort_pin_video"),
|
||||
UniqueConstraint("user_id", "channel_id", name="uq_comfort_pin_channel"),
|
||||
UniqueConstraint("user_id", "plex_item_id", name="uq_comfort_pin_plex_item"),
|
||||
UniqueConstraint("user_id", "plex_show_id", name="uq_comfort_pin_plex_show"),
|
||||
UniqueConstraint("user_id", "playlist_id", name="uq_comfort_pin_playlist"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
video_id: Mapped[str | None] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str | None] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||
plex_item_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_items.id", ondelete="CASCADE")
|
||||
)
|
||||
plex_show_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_shows.id", ondelete="CASCADE")
|
||||
)
|
||||
playlist_id: Mapped[int | None] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
|
||||
|
||||
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
|
||||
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||
|
||||
@@ -33,6 +33,7 @@ def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
||||
"thumb": f"/api/plex/image/{it.rating_key}",
|
||||
"art": _art_url(it.rating_key, it.art_key),
|
||||
"playable": it.playable,
|
||||
"transcode_hw": it.transcode_hw, # HW/SW transcode split → the 4-way corner-dot badge
|
||||
"status": st.status if st else "new",
|
||||
"position_seconds": st.position_seconds if st else 0,
|
||||
}
|
||||
@@ -65,6 +66,7 @@ def episode_card(e: PlexItem, st: PlexState | None) -> dict:
|
||||
"duration_seconds": e.duration_s,
|
||||
"thumb": f"/api/plex/image/{e.rating_key}",
|
||||
"playable": e.playable,
|
||||
"transcode_hw": e.transcode_hw, # HW/SW transcode split → the 4-way corner-dot badge
|
||||
"status": st.status if st else "new",
|
||||
"position_seconds": st.position_seconds if st else 0,
|
||||
}
|
||||
|
||||
@@ -6,16 +6,15 @@ metadata, image proxying, and (P5) optional watch-state write-back. Requests JSO
|
||||
``Accept: application/json`` header (Plex serves XML by default).
|
||||
"""
|
||||
|
||||
import logging
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class PlexError(Exception):
|
||||
@@ -106,7 +105,18 @@ class PlexClient:
|
||||
Best-effort (needs the library's related data) — returns [] if Plex has none."""
|
||||
try:
|
||||
mc = self._get(f"/library/metadata/{rating_key}/related")
|
||||
except PlexError:
|
||||
except PlexError as exc:
|
||||
# An empty list is also the answer for an item that genuinely has no related titles,
|
||||
# so the caller cannot tell the two apart — and it deliberately declines to CACHE an
|
||||
# empty one for exactly that reason. This line is what says which it was.
|
||||
log.debug(
|
||||
"the related items for a Plex item could not be read",
|
||||
extra=logs.event(
|
||||
"plex.client.related_unavailable",
|
||||
rating_key=rating_key,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
return []
|
||||
out: list[dict] = []
|
||||
for hub in mc.get("Hub", []) or []:
|
||||
@@ -139,6 +149,8 @@ class PlexClient:
|
||||
raise PlexError(str(e)) from e
|
||||
try:
|
||||
return (r.json() or {}).get("MediaContainer", {}) or {} if r.content else {}
|
||||
# silent-ok: the HTTP call already succeeded — this is a write endpoint, and Plex answers
|
||||
# those with an empty or non-JSON body as a matter of course.
|
||||
except ValueError:
|
||||
return {}
|
||||
|
||||
@@ -197,7 +209,12 @@ class PlexClient:
|
||||
)
|
||||
|
||||
def unscrobble(self, rating_key: str) -> None:
|
||||
"""Mark the item *unwatched* on the token's Plex account (clears viewCount)."""
|
||||
"""Mark the item *unwatched* on the token's Plex account.
|
||||
|
||||
Clears the view count AND the resume point (`viewOffset`) — measured against the live server
|
||||
on 2026-09-09, from both a watched+offset state and an offset-only one. This one call is the
|
||||
whole un-mark; `/:/timeline?time=0` and `/:/progress?time=0` were both tried there and do
|
||||
NOT clear an offset, so do not reach for them."""
|
||||
self._write(
|
||||
"GET",
|
||||
f"/:/unscrobble?identifier={self._LIBRARY_IDENTIFIER}"
|
||||
|
||||
@@ -22,6 +22,8 @@ def rich_detail(meta: dict) -> dict:
|
||||
if str(r.get("image") or "").startswith("imdb://"):
|
||||
try:
|
||||
imdb_rating = round(float(r.get("value")), 1)
|
||||
# silent-ok: no IMDb score, which is the common and legitimate case — the UI has a
|
||||
# separate audience score to fall back on and shows nothing if neither exists.
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
break
|
||||
|
||||
@@ -11,10 +11,13 @@ The cache directory (``cache.CACHE_DIR``) is shared with the subtitle cache (see
|
||||
from fastapi import Response
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import PlexCollection, PlexItem, PlexSeason, PlexShow
|
||||
from app.plex.cache import CACHE_DIR
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
_EXT_CT = [("jpg", "image/jpeg"), ("png", "image/png"), ("webp", "image/webp")]
|
||||
_CT_EXT = {ct: ext for ext, ct in _EXT_CT}
|
||||
# Target box per variant — Plex resizes to these server-side. Posters fill ≤176px cells (grid/info/
|
||||
@@ -46,6 +49,8 @@ def read(key: str) -> tuple[bytes, str] | None:
|
||||
try:
|
||||
if p.exists() and p.stat().st_size > 0:
|
||||
return p.read_bytes(), ct
|
||||
# silent-ok: a miss. The caller fetches from Plex and writes the file again, and the write
|
||||
# path below is the one that reports a cache that cannot be used.
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
@@ -58,8 +63,14 @@ def write(key: str, data: bytes, ct: str) -> None:
|
||||
tmp = CACHE_DIR / f"{key}.{ext}.tmp"
|
||||
tmp.write_bytes(data)
|
||||
tmp.replace(CACHE_DIR / f"{key}.{ext}") # atomic publish
|
||||
except OSError:
|
||||
pass # cache is best-effort
|
||||
except OSError as exc:
|
||||
# Best-effort, and silently so: with an unwritable cache every poster on every grid goes
|
||||
# to Plex live, for as long as nobody notices. Debug, because a grid is ~100 images and
|
||||
# the point is the condition, not the count.
|
||||
log.debug(
|
||||
"the Plex image cache could not be written",
|
||||
extra=logs.event("plex.imgcache.write_failed", reason=type(exc).__name__),
|
||||
)
|
||||
|
||||
|
||||
def response(data: bytes, ct: str) -> Response:
|
||||
|
||||
@@ -11,5 +11,11 @@ _M = TypeVar("_M")
|
||||
def by_rating_key(db: Session, model: type[_M], rating_key: str) -> _M | None:
|
||||
"""The mirrored row for a Plex rating_key, or None. `model` is any of the rating_key-keyed Plex
|
||||
tables (PlexItem / PlexShow / PlexSeason / PlexCollection); the return type is that model, so call
|
||||
sites keep their `Model | None` static type."""
|
||||
sites keep their `Model | None` static type.
|
||||
|
||||
A key with a NUL in it (`%00` in a path segment) names no row — a 404, like any unknown key.
|
||||
Postgres refuses NUL in text, so asking would be a DataError; the app-wide handler turns the
|
||||
places that DO ask (lists and filters compared to rating_key) into a 422 (see main.py)."""
|
||||
if "\x00" in str(rating_key):
|
||||
return None
|
||||
return db.execute(select(model).where(model.rating_key == str(rating_key))).scalars().first()
|
||||
|
||||
@@ -12,7 +12,9 @@ from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
def parse_map(raw: str) -> list[tuple[str, str]]:
|
||||
@@ -47,6 +49,13 @@ def local_media_path(db: Session, plex_path: str | None) -> Path | None:
|
||||
return None
|
||||
try:
|
||||
p = Path(map_path(db, plex_path)).resolve()
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# None is answered as a 404 "no local file", which is also the answer for a title the
|
||||
# server does not hold — so a dead mount or a wrong path map reads as missing media for
|
||||
# the whole library at once. The path itself stays out: it carries the title.
|
||||
log.warning(
|
||||
"a Plex media path could not be resolved locally",
|
||||
extra=logs.event("plex.paths.resolve_failed", reason=type(exc).__name__),
|
||||
)
|
||||
return None
|
||||
return p if p.is_file() else None
|
||||
|
||||
@@ -8,9 +8,28 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import and_, case, func, or_, select, type_coerce
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import PlexItem, PlexLibrary
|
||||
|
||||
TS_CONFIG = "public.unaccent_simple"
|
||||
|
||||
# The ONE Plex sort vocabulary, shared by every surface that offers the Sort control (the browse grid
|
||||
# and the collection info page) and mirrored by the frontend's `plexSort.ts`. It lives here rather than
|
||||
# inside a router because the two surfaces order DIFFERENT things — browse orders a union subquery in
|
||||
# SQL, a collection orders its already-loaded member list in Python — and a key that one of them cannot
|
||||
# honour would put an option in the UI that silently does nothing. Tests pin both implementations to
|
||||
# this tuple, so adding a key here fails the gate until every surface can order by it.
|
||||
PLEX_SORT_KEYS: tuple[str, ...] = (
|
||||
"added",
|
||||
"lastwatched",
|
||||
"release",
|
||||
"year",
|
||||
"rating",
|
||||
"duration",
|
||||
"title",
|
||||
)
|
||||
|
||||
# --- Language-aware column pickers (E5 S6) ----------------------------------
|
||||
#
|
||||
# The HU-overlay twin (E5 S5) stores Hungarian metadata in `*_hu` columns on the SAME EN row. When
|
||||
@@ -206,6 +225,33 @@ def apply_meta_filters(q, model, p: dict, lang: str = "en"):
|
||||
return q.where(*conds) if conds else q
|
||||
|
||||
|
||||
# --- Playback-class filter (E7) — MOVIE-ONLY (a show is an aggregate with no single class) ---------
|
||||
# The 4 classes the feed filter + facet expose, derived from the stored `playable` + `transcode_hw`.
|
||||
# ONE definition (this SQL CASE) so the filter WHERE and the facet counts can't drift: `playable_conds`
|
||||
# filters on it, `facets` groups by it. transcode_hw IS NOT True groups NULL (unprobed) with software.
|
||||
# Public — it is the shared source of truth imported by routes.plex.browse (facet order + zero-fill).
|
||||
PLAYBACK_CLASSES = ("direct", "remux", "transcode_hw", "transcode_sw")
|
||||
|
||||
|
||||
def playback_class_expr():
|
||||
"""SQL CASE mapping a PlexItem (movie) to its playback-class string, or NULL. The HW branch must
|
||||
precede the generic transcode branch (ordered evaluation)."""
|
||||
return case(
|
||||
(PlexItem.playable == "direct", "direct"),
|
||||
(PlexItem.playable == "remux", "remux"),
|
||||
(and_(PlexItem.playable == "transcode", PlexItem.transcode_hw.is_(True)), "transcode_hw"),
|
||||
(PlexItem.playable == "transcode", "transcode_sw"),
|
||||
else_=None,
|
||||
)
|
||||
|
||||
|
||||
def playable_conds(sel: list[str]) -> list:
|
||||
"""A one-element condition list (or empty) selecting the given playback classes — an OR via
|
||||
`class IN (...)` over the shared CASE. Unknown keys are dropped."""
|
||||
keep = [c for c in sel if c in PLAYBACK_CLASSES]
|
||||
return [playback_class_expr().in_(keep)] if keep else []
|
||||
|
||||
|
||||
@dataclass
|
||||
class LibraryFilters:
|
||||
"""The shared sidebar filter query-params for /library + /facets, declared once and injected via
|
||||
@@ -228,6 +274,7 @@ class LibraryFilters:
|
||||
keywords: str | None = None # TMDB keyword chips (E5 S5), CSV; language-neutral
|
||||
collections: str | None = None # CSV of collection rating_keys
|
||||
collection_mode: str = "any" # how multiple collections combine: any (OR, default) | all (AND)
|
||||
playable: str | None = None # CSV of playback classes: direct|remux|transcode_hw|transcode_sw
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {
|
||||
@@ -247,13 +294,10 @@ class LibraryFilters:
|
||||
"keywords": self.keywords,
|
||||
"collections": self.collections,
|
||||
"collection_mode": self.collection_mode,
|
||||
"playable": self.playable,
|
||||
}
|
||||
|
||||
|
||||
def like_escape(s: str) -> str:
|
||||
return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
# --- Watch-state scope (shared by the /library grid AND the /facets aggregation) ---------------
|
||||
#
|
||||
# The `show` scope (all | watched | in_progress | unwatched) filters the browse result by per-user
|
||||
@@ -311,3 +355,26 @@ def apply_show_state_filter(query, tot, wat, inp, show: str):
|
||||
if show == "unwatched":
|
||||
return query.where(wat == 0, inp == 0)
|
||||
return query
|
||||
|
||||
|
||||
def content_libs(db: Session) -> tuple[list[int], list[int]]:
|
||||
"""The BROWSABLE content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
|
||||
source of the browse scope. Excludes HU overlay libraries (`overlay_of` set): those are never
|
||||
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows (E5 S5).
|
||||
|
||||
Lives here, in the query layer, rather than in `routes/plex/_shared.py` where it started: E24's
|
||||
Home surface needs the same scope, and a domain package importing from `routes/` would invert the
|
||||
one-way dependency CLAUDE.md draws. It never touched request state, so nothing was lost by the
|
||||
move."""
|
||||
libs = (
|
||||
db.execute(
|
||||
select(PlexLibrary).where(
|
||||
PlexLibrary.enabled.is_(True), PlexLibrary.overlay_of.is_(None)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
movie_lib_ids = [lb.id for lb in libs if lb.kind == "movie"]
|
||||
show_lib_ids = [lb.id for lb in libs if lb.kind == "show"]
|
||||
return movie_lib_ids, show_lib_ids
|
||||
|
||||
@@ -3,11 +3,25 @@
|
||||
Drives the show/season grid badges and the Resume / Play-from-start / mark-all buttons.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.models import PlexItem, PlexState
|
||||
|
||||
# Sort sentinel for a rail row whose timestamp is missing, so `key=lambda r: r[0] or NO_TS` reads the
|
||||
# same at every rail that merges two type-homogeneous lists in Python.
|
||||
#
|
||||
# It is NOT a crash fix, whatever the E24 commit that introduced it says. The `(ts is not None, ts)`
|
||||
# key it replaced cannot raise: tuple comparison finds the first UNEQUAL element via `==`, so two
|
||||
# `(False, None)` keys compare equal and `None < None` is never reached — verified by execution
|
||||
# during the E24 epic-end review, over every permutation of null/non-null rows. What the sentinel
|
||||
# buys is one idiom instead of two, and a key that says "missing sorts last" outright rather than
|
||||
# encoding it in a boolean flag. A BARE `r[0]` would genuinely raise, which is the mutation the
|
||||
# rails' tests do guard against.
|
||||
NO_TS = datetime.min.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def show_status(total: int, watched: int, inprog: int) -> str:
|
||||
"""Roll a show's episodes up into a single watch state: fully watched, partially started, or new."""
|
||||
@@ -29,7 +43,9 @@ def rollup(cards: list[dict]) -> dict:
|
||||
]
|
||||
resume = None
|
||||
if inprog:
|
||||
resume = inprog[-1] # continue the latest-started episode
|
||||
# The LAST in-progress episode in broadcast order — no timestamp is consulted. Home's
|
||||
# `discover.queries.next_episode` picks the same one, so both Resume buttons agree.
|
||||
resume = inprog[-1]
|
||||
else:
|
||||
resume = next((c for c in cards if c.get("status") != "watched"), None)
|
||||
return {
|
||||
@@ -43,8 +59,10 @@ def rollup(cards: list[dict]) -> dict:
|
||||
def agg_subquery(db: Session, user_id: int, include_resumed: bool = False):
|
||||
"""Per-show episode counts for a user: total, watched, in-progress. Reused to (a) filter the show
|
||||
grid by aggregate watch-state, (b) compute each card's badge, and (c) drive the Discover
|
||||
continue-watching rail. `include_resumed` adds `resumed_at` = max(progress_updated_at) across the
|
||||
show's episodes, so the rail can order shows by how recently they were last progressed."""
|
||||
continue-watching rail, and (d) the browse "last watched" sort. `include_resumed` adds `resumed_at`
|
||||
= max(coalesce(progress_updated_at, watched_at)) across the show's episodes — i.e. how recently ANY
|
||||
episode was last touched (resumed OR marked watched) — so callers can order shows by recency. A
|
||||
fully-watched show (no resume progress) still gets a timestamp from its episodes' watched_at."""
|
||||
ps = aliased(PlexState)
|
||||
cols = [
|
||||
PlexItem.show_id.label("show_id"),
|
||||
@@ -63,7 +81,9 @@ def agg_subquery(db: Session, user_id: int, include_resumed: bool = False):
|
||||
).label("inprog"),
|
||||
]
|
||||
if include_resumed:
|
||||
cols.append(func.max(ps.progress_updated_at).label("resumed_at"))
|
||||
cols.append(
|
||||
func.max(func.coalesce(ps.progress_updated_at, ps.watched_at)).label("resumed_at")
|
||||
)
|
||||
return (
|
||||
select(*cols)
|
||||
.outerjoin(ps, and_(ps.item_id == PlexItem.id, ps.user_id == user_id))
|
||||
|
||||
+796
-77
File diff suppressed because it is too large
Load Diff
+893
-18
@@ -1,16 +1,58 @@
|
||||
"""WebVTT conversion, HLS-clock time-shift, and on-disk cache for Plex subtitle streams.
|
||||
"""WebVTT conversion, HLS-clock time-shift, and the on-disk cache + background extractor for Plex
|
||||
subtitle streams.
|
||||
|
||||
Text subtitles (embedded or external) are turned into WebVTT so the browser <track> can render them;
|
||||
bitmap subtitles (Blu-ray/DVD) can't be (would need burn-in). Converted VTTs are cached on the shared
|
||||
Plex on-disk cache root (``cache.CACHE_DIR``).
|
||||
|
||||
WHY THERE IS A BACKGROUND EXTRACTOR HERE — measured on prod, 2026-09-10, against a 17.06 GB 1080p
|
||||
MKV with two embedded SubRip tracks on the 81 TB array:
|
||||
|
||||
=========================================================== ========
|
||||
full pass, ONE track (what this module used to do) 154 s
|
||||
full pass, BOTH text tracks in a SINGLE ffmpeg run 111 s
|
||||
`-c:s copy` instead of converting to WebVTT 143 s
|
||||
from 1:00:00 to the end of the file 66 s
|
||||
a 630 s WINDOW at 1:00:00, both tracks 8 s
|
||||
=========================================================== ========
|
||||
|
||||
The conversion is not the cost (dropping it saved 7%): ffmpeg has to stream every byte of the
|
||||
container between its seek point and its stop point, and the array delivers ~140 MB/s — so the time
|
||||
is (bytes read) / 140 MB/s, and the ONLY lever is reading less. Hence:
|
||||
|
||||
* extract a WINDOW around the playhead first — the viewer sees subtitles in seconds — then complete
|
||||
the whole track in the background so every later seek is instant and the next session is free;
|
||||
* extract EVERY embedded text track in ONE pass, because the read is shared: picking Hungarian used
|
||||
to cost a full read and then switching to English cost another one (the "hektikus" report);
|
||||
* never do any of it inside a request (it froze the <track> with no feedback AND held an API worker
|
||||
thread for the whole read).
|
||||
|
||||
`-copyts` CANNOT be used for a window: it makes ffmpeg ignore `-t`/`-to` (measured — a [0, 600 s]
|
||||
window still took the full 145 s and emitted the whole track), so windows are extracted on a
|
||||
zero-based clock and shifted back to absolute time here with `shift_vtt`.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.db import SessionLocal
|
||||
from app.models import PlexItem
|
||||
from app.plex import paths as plex_paths
|
||||
from app.plex.cache import CACHE_DIR
|
||||
from app.plex.client import PlexError
|
||||
from app.plex.client import PlexClient, PlexError
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Image-based subtitle codecs (Blu-ray/DVD) — can't be turned into text WebVTT (would need burn-in).
|
||||
BITMAP_SUBS = {
|
||||
"pgs",
|
||||
@@ -31,6 +73,9 @@ def srt_to_vtt(text: str) -> str:
|
||||
|
||||
|
||||
def _vtt_ts_to_s(ts: str) -> float:
|
||||
# Also reads ffmpeg's spelling of a NEGATIVE time (a windowed pass, see `_ffmpeg_extract`): its
|
||||
# webvtt muxer splits the milliseconds with C's truncating division, so every field carries the
|
||||
# sign — -5.005 s is `00:-5.-05`, -65.3 s is `-1:-5.-300` — and summing the signed fields is exact.
|
||||
parts = ts.split(":")
|
||||
h = "0" if len(parts) == 2 else parts[0]
|
||||
m, rest = parts[-2], parts[-1]
|
||||
@@ -39,16 +84,12 @@ def _vtt_ts_to_s(ts: str) -> float:
|
||||
|
||||
|
||||
def _vtt_s_to_ts(x: float) -> str:
|
||||
x = max(0.0, x)
|
||||
h = int(x // 3600)
|
||||
x -= h * 3600
|
||||
m = int(x // 60)
|
||||
x -= m * 60
|
||||
s = int(x)
|
||||
ms = int(round((x - s) * 1000))
|
||||
if ms >= 1000:
|
||||
s += 1
|
||||
ms -= 1000
|
||||
# Rounded to whole milliseconds FIRST, then split: rounding the fraction last carried a rounded-up
|
||||
# millisecond into the seconds and no further, so 59.9996 came out as `00:00:60.000` — a timestamp
|
||||
# the WebVTT parser rejects, silently dropping that cue.
|
||||
h, rest = divmod(int(round(max(0.0, x) * 1000)), 3_600_000)
|
||||
m, rest = divmod(rest, 60_000)
|
||||
s, ms = divmod(rest, 1000)
|
||||
return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}"
|
||||
|
||||
|
||||
@@ -63,6 +104,7 @@ def shift_vtt(vtt: str, offset: float) -> str:
|
||||
straddling 0 is clamped to start at 0."""
|
||||
if not offset:
|
||||
return vtt
|
||||
unshifted = 0
|
||||
out_blocks: list[str] = []
|
||||
for block in re.split(r"\n[ \t]*\n", vtt):
|
||||
if "-->" not in block:
|
||||
@@ -80,7 +122,12 @@ def shift_vtt(vtt: str, offset: float) -> str:
|
||||
st = _vtt_ts_to_s(left.strip()) - offset
|
||||
en = _vtt_ts_to_s(rparts[0]) - offset
|
||||
except (ValueError, IndexError):
|
||||
break # unparseable timing → leave the block as-is
|
||||
# silent-ok: left as-is means left UNSHIFTED — the viewer sees that one cue at
|
||||
# the wrong moment. Counted rather than logged here: this is per cue, and the file
|
||||
# being parsed is one ffmpeg wrote for us a moment ago, so a broken timestamp is
|
||||
# a fact about the extraction, reported once below.
|
||||
unshifted += 1
|
||||
break
|
||||
if en <= 0:
|
||||
keep = False # whole cue is before the session start → drop the block
|
||||
else:
|
||||
@@ -90,6 +137,11 @@ def shift_vtt(vtt: str, offset: float) -> str:
|
||||
break
|
||||
if keep:
|
||||
out_blocks.append("\n".join(lines))
|
||||
if unshifted:
|
||||
log.info(
|
||||
"some cues could not be shifted and stay at their original time",
|
||||
extra=logs.event("plex.subtitles.cues_unshifted", cues=unshifted, offset_s=offset),
|
||||
)
|
||||
return "\n\n".join(out_blocks)
|
||||
|
||||
|
||||
@@ -109,7 +161,7 @@ def ffmpeg_to_vtt(src: str, sub_ord: int | None) -> str:
|
||||
"webvtt",
|
||||
"pipe:1",
|
||||
]
|
||||
r = subprocess.run(cmd, capture_output=True)
|
||||
r = subprocess.run(cmd, capture_output=True, timeout=EXTRACT_TIMEOUT_S)
|
||||
if r.returncode != 0 or not r.stdout:
|
||||
raise PlexError(
|
||||
(r.stderr or b"").decode("utf-8", "replace")[:200] or "ffmpeg subtitle convert failed"
|
||||
@@ -121,15 +173,838 @@ def cache_read(key: str) -> str | None:
|
||||
try:
|
||||
p = CACHE_DIR / f"{key}.vtt"
|
||||
return p.read_text(encoding="utf-8") if p.exists() else None
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# None is the same answer as "never cached", so a cache we cannot read looks exactly like
|
||||
# a cold one — and every request re-extracts instead of saying the disk is the problem.
|
||||
log.warning(
|
||||
"the cached subtitle could not be read",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.cache_read_failed", key=key, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def cache_write(key: str, vtt: str) -> None:
|
||||
def cache_write(key: str, vtt: str) -> bool:
|
||||
"""Atomically replace the cached VTT; False when it could not be written (`publish` must then
|
||||
not claim the region it was carrying)."""
|
||||
try:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CACHE_DIR / f"{key}.vtt.tmp"
|
||||
tmp.write_text(vtt, encoding="utf-8")
|
||||
tmp.replace(CACHE_DIR / f"{key}.vtt") # atomic publish
|
||||
except OSError:
|
||||
pass
|
||||
return True
|
||||
except OSError as exc:
|
||||
# `publish` turns False into "the region was not claimed" and moves on, so the track just
|
||||
# never appears. Which errno it was — full disk, read-only mount, permissions — is the
|
||||
# whole diagnosis, and it exists only here.
|
||||
log.warning(
|
||||
"the subtitle cache could not be written",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.cache_write_failed", key=key, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
# --- cache identity -----------------------------------------------------------------------------
|
||||
|
||||
# The cached VTT holds ABSOLUTE cue times (per rating_key + subtitle ordinal); the per-session shift
|
||||
# is applied on the way out, so a resume/seek to any offset reuses the same file. Beside it, a small
|
||||
# JSON sidecar records WHICH absolute-time ranges the file actually covers — a windowed extraction
|
||||
# publishes a usable but incomplete track, and "incomplete" must never be mistaken for "all there is".
|
||||
_FULL_END_S = 1_000_000_000.0 # stand-in for "to the end of the file" (JSON can't hold infinity)
|
||||
|
||||
|
||||
def cache_key(rating_key: str, sub_ord: int) -> str:
|
||||
return f"sub_{rating_key}_{sub_ord}"
|
||||
|
||||
|
||||
def _vtt_path(key: str) -> Path:
|
||||
return CACHE_DIR / f"{key}.vtt"
|
||||
|
||||
|
||||
def _meta_path(key: str) -> Path:
|
||||
return CACHE_DIR / f"{key}.json"
|
||||
|
||||
|
||||
def read_meta(key: str) -> tuple[list[list[float]], bool]:
|
||||
"""Covered absolute-time ranges + whether the track is fully extracted. A VTT with no sidecar
|
||||
(written by an older build, which only ever cached complete tracks) counts as complete."""
|
||||
try:
|
||||
raw = json.loads(_meta_path(key).read_text(encoding="utf-8"))
|
||||
return [[float(a), float(b)] for a, b in raw.get("ranges", [])], bool(raw.get("complete"))
|
||||
# silent-ok: no sidecar at all is the documented older-build case the fallback exists for.
|
||||
except FileNotFoundError:
|
||||
return ([[0.0, _FULL_END_S]], True) if _exists(_vtt_path(key)) else ([], False)
|
||||
except (OSError, ValueError, TypeError) as exc:
|
||||
# A sidecar that EXISTS and will not parse is different, and the fallback is the dangerous
|
||||
# answer to it: beside a cached VTT it reports the track as complete, so a partial track is
|
||||
# never extended again and the viewer loses every cue past the window that was cached.
|
||||
log.warning(
|
||||
"the subtitle sidecar could not be read; the cached track is taken as complete",
|
||||
extra=logs.event("plex.subtitles.meta_unreadable", key=key, reason=type(exc).__name__),
|
||||
)
|
||||
return ([[0.0, _FULL_END_S]], True) if _exists(_vtt_path(key)) else ([], False)
|
||||
|
||||
|
||||
def _exists(p: Path) -> bool:
|
||||
"""`Path.exists` that answers False for a path the OS refuses outright: pathlib swallows only
|
||||
ENOENT-like errors, so an over-long name (ENAMETOOLONG, from an absurd rating_key or ordinal)
|
||||
escaped as a 500 from the status route."""
|
||||
try:
|
||||
return p.exists()
|
||||
# silent-ok: this function's whole job is to answer False for a path the OS refuses. The
|
||||
# refusal IS the return value; a caller that needs to report one reports it itself.
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _write_meta(key: str, ranges: list[list[float]], complete: bool) -> None:
|
||||
try:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CACHE_DIR / f"{key}.json.tmp"
|
||||
tmp.write_text(json.dumps({"ranges": ranges, "complete": complete}), encoding="utf-8")
|
||||
tmp.replace(_meta_path(key))
|
||||
except OSError as exc:
|
||||
# Without the sidecar beside it, a cached window reads back as a COMPLETE track (see
|
||||
# `read_meta`) — so one failed write here permanently costs the rest of that subtitle.
|
||||
log.warning(
|
||||
"the subtitle sidecar could not be written",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.meta_write_failed",
|
||||
key=key,
|
||||
complete=complete,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def generation(key: str) -> int:
|
||||
"""Monotonic id of the cached file's current content. It goes into the <track> URL so that when a
|
||||
window is superseded by a wider one the browser refetches instead of keeping the short VTT."""
|
||||
try:
|
||||
return _vtt_path(key).stat().st_mtime_ns
|
||||
# silent-ok: 0 for a track that is not cached yet, which is the ordinary case on every first
|
||||
# request. (ValueError: a NUL in the key — `stat` refuses the path outright.)
|
||||
except (OSError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def merge_ranges(ranges: list[list[float]]) -> list[list[float]]:
|
||||
"""Sort + coalesce touching/overlapping covered ranges."""
|
||||
out: list[list[float]] = []
|
||||
for a, b in sorted((r for r in ranges if r[1] > r[0]), key=lambda r: r[0]):
|
||||
if out and a <= out[-1][1]:
|
||||
out[-1][1] = max(out[-1][1], b)
|
||||
else:
|
||||
out.append([a, b])
|
||||
return out
|
||||
|
||||
|
||||
def covers(ranges: list[list[float]], at: float) -> bool:
|
||||
return any(a <= at <= b for a, b in ranges)
|
||||
|
||||
|
||||
def merge_vtt(*sources: str) -> str:
|
||||
"""Union of the cue blocks of several absolute-time VTTs, ordered by start time.
|
||||
|
||||
Two extraction passes over the same file emit identical cues for the region they share, so they
|
||||
are de-duplicated — but NOT on the raw timing text. The two passes do not spell a timestamp the
|
||||
same way: a window comes back through `shift_vtt`, which always writes `HH:MM:SS.mmm`, while a
|
||||
full pass is ffmpeg's own output, and its webvtt muxer drops the hour below one hour
|
||||
(`10:00.123`). Keyed on the text, every cue of a window in the first hour survived twice and the
|
||||
viewer saw each line stacked on itself. So each timing line is parsed and re-emitted in one
|
||||
canonical form, and the key is that canonical timing PLUS the cue text: two different cues that
|
||||
happen to share a time (two speakers, a sign over dialogue) are both kept."""
|
||||
seen: set[str] = set()
|
||||
cues: list[tuple[float, str]] = []
|
||||
dropped = 0
|
||||
for src in sources:
|
||||
for block in re.split(r"\n[ \t]*\n", src or ""):
|
||||
stripped = block.strip()
|
||||
canon = _canonical_cue(stripped)
|
||||
if canon is None:
|
||||
# A block WITH a timing line that still would not parse is a cue we are throwing
|
||||
# away, not a header — and the merged track would simply be missing those lines.
|
||||
if "-->" in stripped:
|
||||
dropped += 1
|
||||
continue # header / NOTE / STYLE — the merged file gets its own single header
|
||||
start, key, text = canon
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
cues.append((start, text))
|
||||
if dropped:
|
||||
log.info(
|
||||
"some cues were dropped from the merged subtitle: their timing would not parse",
|
||||
extra=logs.event("plex.subtitles.cues_dropped", cues=dropped, kept=len(cues)),
|
||||
)
|
||||
cues.sort(key=lambda c: c[0])
|
||||
return "WEBVTT\n\n" + "\n\n".join(b for _, b in cues) + "\n"
|
||||
|
||||
|
||||
def _canonical_cue(block: str) -> tuple[float, str, str] | None:
|
||||
"""(start seconds, identity, the block with its timing line canonicalised) — or None for a block
|
||||
that is not a cue. The identity leaves out anything before the timing line (an SRT-style cue
|
||||
number), which is a label, not content."""
|
||||
lines = block.split("\n")
|
||||
for i, ln in enumerate(lines):
|
||||
if "-->" not in ln:
|
||||
continue
|
||||
left, _, right = ln.partition("-->")
|
||||
rparts = right.strip().split(None, 1)
|
||||
settings = (" " + rparts[1]) if len(rparts) > 1 else ""
|
||||
try:
|
||||
st = _vtt_ts_to_s(left.strip())
|
||||
en = _vtt_ts_to_s(rparts[0])
|
||||
# silent-ok: None here means "not a cue", and the caller tells a header apart from a
|
||||
# broken timing line by whether the block had a `-->` at all — see `merge_vtt`.
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
lines[i] = f"{_vtt_s_to_ts(st)} --> {_vtt_s_to_ts(en)}{settings}"
|
||||
return st, "\n".join(lines[i:]), "\n".join(lines)
|
||||
return None
|
||||
|
||||
|
||||
def _last_cue_end(vtt: str) -> float:
|
||||
"""Where the last cue of a VTT ends — how far a pass that died early provably got."""
|
||||
end = 0.0
|
||||
for ln in vtt.split("\n"):
|
||||
if "-->" not in ln:
|
||||
continue
|
||||
try:
|
||||
end = max(end, _vtt_ts_to_s(ln.partition("-->")[2].strip().split(None, 1)[0]))
|
||||
# silent-ok: this is a floor — "how far the pass provably got". Skipping a line it cannot
|
||||
# read makes the answer smaller, never larger, so the error is always on the safe side.
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
return end
|
||||
|
||||
|
||||
def publish(key: str, vtt: str, covered: tuple[float, float], complete: bool) -> bool:
|
||||
"""Merge a freshly extracted region into the cached track and widen its covered ranges. False
|
||||
when the region did not land (the cache is unwritable, or the VTT there cannot be read) — the
|
||||
caller must then record a FAILED pass: returning quietly let the next poll schedule the same read
|
||||
again, a whole-container pass every time, while the player said "preparing" for good.
|
||||
|
||||
A read-merge-write, so it runs under `_PUBLISH_LOCK`: two jobs publish into the same keys (a
|
||||
window and the full pass, and every embedded track of the file rides each of them), and without
|
||||
the lock the later writer drops whatever the earlier one merged — permanently, once `complete` is
|
||||
set. It is also the only writer of the cache files, which is what keeps their fixed `.tmp`
|
||||
staging names safe."""
|
||||
with _PUBLISH_LOCK:
|
||||
old = cache_read(key)
|
||||
if old is None and _vtt_path(key).exists():
|
||||
# There, but unreadable (`cache_read` swallows the OSError). Merging into "" would replace
|
||||
# a whole track with this one region while its sidecar still claimed all of it.
|
||||
log.warning("subtitle cache %s unreadable; not publishing into it", key)
|
||||
return False
|
||||
old_ranges, was_complete = read_meta(key)
|
||||
if old is None:
|
||||
# A sidecar with no VTT behind it claims cues that are not there, so it is not merged
|
||||
# into. And a VTT with no sidecar reads as a COMPLETE track (see read_meta — how older
|
||||
# builds cached), while a publish is two renames: so before a key's first VTT lands, its
|
||||
# sidecar is made to claim nothing. Dying between the two then leaves a window that is
|
||||
# extracted again, never one taken for the whole track for good.
|
||||
old_ranges, was_complete = [], False
|
||||
_write_meta(key, [], False)
|
||||
if not cache_write(key, merge_vtt(old or "", vtt)):
|
||||
log.warning("subtitle cache %s could not be written", key)
|
||||
return False # the region never landed, so the sidecar must not claim it
|
||||
_write_meta(key, merge_ranges(old_ranges + [list(covered)]), complete or was_complete)
|
||||
return True
|
||||
|
||||
|
||||
# --- background extraction ----------------------------------------------------------------------
|
||||
|
||||
# How much of the track to extract before the viewer sees anything. WINDOW_PRE_S starts a little
|
||||
# before the playhead so a cue that is already on screen is not missed; WINDOW_LEN_S is the span
|
||||
# published as the first, provisional VTT (~4 s of work on the measured file). It only has to hold
|
||||
# until the full pass lands behind it — 111 s for that file — so it is deliberately much longer than
|
||||
# any realistic full pass.
|
||||
WINDOW_PRE_S = 30.0
|
||||
WINDOW_LEN_S = 300.0
|
||||
# …but it does not always land: a full pass can be backed off (FULL_RETRY_S) or simply slower than
|
||||
# the window lasts (a file past ~40 GB, or several passes sharing the array). A window was only asked
|
||||
# for once the playhead had LEFT coverage, so every edge unmounted the track for a poll plus an
|
||||
# extraction. Within this distance of a covered stretch's end, the next window is read ahead.
|
||||
LOOKAHEAD_S = 60.0
|
||||
# Ceiling on one ffmpeg run so a wedged extraction can't hold a pool thread forever (the old
|
||||
# in-request extraction had no timeout at all).
|
||||
EXTRACT_TIMEOUT_S = 900.0
|
||||
# How long a timed-out ffmpeg gets, after a TERM, to close its outputs before it is killed.
|
||||
_TERM_GRACE_S = 10.0
|
||||
# A failed track (or window) is remembered briefly so a 1 s status poll doesn't respawn ffmpeg every
|
||||
# second, and forgotten again soon after so fixing the file (or Plex coming back) doesn't need a
|
||||
# restart.
|
||||
FAIL_TTL_S = 60.0
|
||||
# …except a FULL pass that failed. That is a read of the whole container (minutes), a failure that
|
||||
# reproduces does so every time, and the viewer is typically inside what it did publish — so the
|
||||
# short memo re-armed a whole-file read every minute for as long as they watched. Windows around the
|
||||
# playhead still run meanwhile; only the full pass waits this out.
|
||||
FULL_RETRY_S = 3600.0
|
||||
# The furthest position the status poll accepts. No film is this long, and an absurd one used to
|
||||
# reach ffmpeg's `-ss` (which rejects anything past its int64 microseconds).
|
||||
MAX_POSITION_S = 100_000.0
|
||||
# Windows and full passes run on SEPARATE pools, so a window never waits behind a full pass. They
|
||||
# shared one FIFO pool of three whose comment promised "room for a window" that nothing enforced:
|
||||
# stepping through three episodes with subtitles on left three full passes (minutes each) holding every
|
||||
# worker, and a seek window on the third queued behind them. Two of each: bounded, so a burst can't
|
||||
# thrash the array.
|
||||
_WIN_POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="plex-subs-win")
|
||||
_FULL_POOL = ThreadPoolExecutor(max_workers=2, thread_name_prefix="plex-subs-full")
|
||||
# Bounds on what may wait for those pools — separate too, so held full-pass slots (minutes each)
|
||||
# cannot refuse a window (seconds). The queues are unbounded and every job is a read of the library:
|
||||
# a viewer scrubbing through a film queued a window per stop (the one where they stopped ran LAST),
|
||||
# and a script could queue thousands. Past a bound nothing is queued — the next poll asks again, by
|
||||
# which time something has finished.
|
||||
_MAX_WINDOWS_INFLIGHT = 8
|
||||
_MAX_FULL_INFLIGHT = 6
|
||||
# …of which one account may hold this many. The full-pass slots are minutes each, and a hostile (or
|
||||
# merely curious) account opening subtitles on film after film could otherwise hold all of them while
|
||||
# everyone else's tracks never completed. A viewer over their share still gets windows — the part
|
||||
# that makes a track watchable now — just not another whole-file read until one of theirs ends.
|
||||
_MAX_FULL_PER_USER = 3
|
||||
_MAX_TRACK_WINDOWS = 2
|
||||
|
||||
_LOCK = threading.Lock()
|
||||
# Queued or running jobs → the absolute span a window job reads (None for a full pass).
|
||||
_INFLIGHT: dict[tuple, tuple[float, float] | None] = {}
|
||||
# rating_key → the account whose poll queued that item's full pass (see _MAX_FULL_PER_USER).
|
||||
_FULL_OWNERS: dict[str, int] = {}
|
||||
# Track-level failures: a track that cannot be served at all (no item, a bitmap track, Plex down…).
|
||||
_FAILURES: dict[str, tuple[str, float]] = {}
|
||||
# rating_key → when its full pass last failed (see FULL_RETRY_S).
|
||||
_FULL_FAILED: dict[str, float] = {}
|
||||
# cache key → the spans windows failed to read, with when. A failed window says nothing about the rest
|
||||
# of the track — the full pass may be about to deliver it — so it is remembered as a SPAN: it must be
|
||||
# neither respawned every poll nor taken as a failure of positions it never tried. (It used to set the
|
||||
# track-level memo, which told every viewer at any uncovered position "failed" — a final answer that
|
||||
# ends the player's poll — including one planted by a request for an absurd position.)
|
||||
_WIN_FAILED: dict[str, list[tuple[float, float, float]]] = {}
|
||||
_PUBLISH_LOCK = threading.Lock()
|
||||
# The ffmpeg children currently running, so a shutdown can kill them (see `shutdown`), and whether it
|
||||
# has: a worker that reaches its Popen after the kill sweep must not start a read.
|
||||
_PROCS: set[subprocess.Popen] = set()
|
||||
_STOPPING = False
|
||||
|
||||
|
||||
class SubtitleUnavailable(Exception):
|
||||
"""A track that cannot be served as text, carrying the stable code the route maps to a status."""
|
||||
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
# code -> HTTP status. An extraction/upstream failure is 422 with a structured detail (502 is
|
||||
# reserved for the raw byte proxies); the rest are genuine "no such thing" / "not text".
|
||||
UNAVAILABLE_STATUS = {
|
||||
"no_item": 404,
|
||||
"no_track": 404,
|
||||
"no_file": 404,
|
||||
"not_fetchable": 404,
|
||||
"bitmap": 415,
|
||||
"extract_failed": 422,
|
||||
"plex_error": 422,
|
||||
}
|
||||
# The codes that are a FINAL answer about the track: it can never be served as text, so the player
|
||||
# stops asking. The others (`plex_error`, `extract_failed`) are about now or about one stretch — a
|
||||
# memo that lapses, a window that failed where the rest is fine — and answered as `unavailable`: said,
|
||||
# but asked again. Given as `failed`, one Plex hiccup ended the subtitles for the rest of the session,
|
||||
# at cached positions too, and after the full pass had completed behind it.
|
||||
FINAL_CODES = frozenset({"no_item", "no_track", "no_file", "not_fetchable", "bitmap"})
|
||||
|
||||
|
||||
def reset_state() -> None:
|
||||
"""Drop the in-flight/failure registries (tests; nothing in the app needs it)."""
|
||||
global _STOPPING
|
||||
with _LOCK:
|
||||
_INFLIGHT.clear()
|
||||
_FULL_OWNERS.clear()
|
||||
_FAILURES.clear()
|
||||
_FULL_FAILED.clear()
|
||||
_WIN_FAILED.clear()
|
||||
_STOPPING = False
|
||||
|
||||
|
||||
def shutdown() -> None:
|
||||
"""Stop extracting, for the app's shutdown. The pool's workers are non-daemon threads and Python
|
||||
joins them at interpreter exit — so a worker still inside a full pass (minutes on a big file, up
|
||||
to EXTRACT_TIMEOUT_S) held the process open after uvicorn had stopped, until the container runtime
|
||||
SIGKILLed it. Queued jobs are dropped and running ffmpeg children are killed, which returns their
|
||||
workers at once — and a worker still BEFORE its ffmpeg (the Plex lookup, the DB) finds the flag
|
||||
and never starts one. Nothing that matters is lost: an interrupted publish cannot pass for a whole
|
||||
track (see `publish`), and an interrupted track is simply extracted again on the next poll."""
|
||||
global _STOPPING
|
||||
with _LOCK:
|
||||
_STOPPING = True
|
||||
procs = list(_PROCS)
|
||||
for pool in (_WIN_POOL, _FULL_POOL):
|
||||
pool.shutdown(wait=False, cancel_futures=True)
|
||||
for p in procs:
|
||||
p.kill()
|
||||
|
||||
|
||||
def _sweep_memos() -> None:
|
||||
"""(Caller holds _LOCK.) Drop every lapsed failure memo. They were only ever pruned when the SAME
|
||||
key was read again, and the status route takes any rating_key/ordinal — so a script polling fresh
|
||||
keys grew the API process without bound. Swept on every write, they hold at most a TTL's worth."""
|
||||
now = time.monotonic()
|
||||
for k in [k for k, (_, t) in _FAILURES.items() if now - t >= FAIL_TTL_S]:
|
||||
del _FAILURES[k]
|
||||
for k in [k for k, t in _FULL_FAILED.items() if now - t >= FULL_RETRY_S]:
|
||||
del _FULL_FAILED[k]
|
||||
for k in list(_WIN_FAILED):
|
||||
_live_window_failures(k)
|
||||
|
||||
|
||||
def _fail(key: str, code: str) -> None:
|
||||
with _LOCK:
|
||||
_sweep_memos()
|
||||
_FAILURES[key] = (code, time.monotonic())
|
||||
|
||||
|
||||
def _recent_failure(key: str) -> str | None:
|
||||
with _LOCK:
|
||||
hit = _FAILURES.get(key)
|
||||
if hit and time.monotonic() - hit[1] < FAIL_TTL_S:
|
||||
return hit[0]
|
||||
_FAILURES.pop(key, None)
|
||||
return None
|
||||
|
||||
|
||||
def _full_failed(rating_key: str) -> bool:
|
||||
with _LOCK:
|
||||
when = _FULL_FAILED.get(rating_key)
|
||||
if when is not None and time.monotonic() - when < FULL_RETRY_S:
|
||||
return True
|
||||
_FULL_FAILED.pop(rating_key, None)
|
||||
return False
|
||||
|
||||
|
||||
def _live_window_failures(key: str) -> list[tuple[float, float, float]]:
|
||||
"""(Caller holds _LOCK.) The key's failed-window spans that are still remembered."""
|
||||
now = time.monotonic()
|
||||
live = [f for f in _WIN_FAILED.get(key, []) if now - f[2] < FAIL_TTL_S]
|
||||
if live:
|
||||
_WIN_FAILED[key] = live
|
||||
else:
|
||||
_WIN_FAILED.pop(key, None)
|
||||
return live
|
||||
|
||||
|
||||
def _window_fail(key: str, span: tuple[float, float]) -> None:
|
||||
with _LOCK:
|
||||
_sweep_memos()
|
||||
_WIN_FAILED[key] = [*_live_window_failures(key), (span[0], span[1], time.monotonic())]
|
||||
|
||||
|
||||
def _window_failed(key: str, at: float) -> bool:
|
||||
with _LOCK:
|
||||
return any(a <= at <= b for a, b, _ in _live_window_failures(key))
|
||||
|
||||
|
||||
def ensure(rating_key: str, sub_ord: int, at: float = 0.0, viewer: int = 0) -> dict:
|
||||
"""State of one subtitle track, scheduling whatever extraction is still missing. NEVER blocks.
|
||||
|
||||
``ready`` = fully extracted · ``partial`` = usable, covers ``at``, still growing · ``preparing``
|
||||
= nothing to show for ``at`` yet · ``unavailable`` = nothing for ``at`` and nothing being read
|
||||
for it right now (``code`` says why), ask again · ``failed`` = the track can never be shown
|
||||
(``code`` says why) — the only final answer. ``gen`` changes whenever the cached file grows, so
|
||||
the client can refetch a window that has been superseded; 0 = nothing is cached at all."""
|
||||
key = cache_key(rating_key, sub_ord)
|
||||
ranges, complete = read_meta(key)
|
||||
have = cache_read(key) is not None
|
||||
covered = have and covers(ranges, at)
|
||||
gen = generation(key)
|
||||
if complete and have:
|
||||
return {"state": "ready", "code": None, "gen": gen}
|
||||
if not have and _exists(_vtt_path(key)):
|
||||
# On disk but unreadable (permissions, a bad volume): nothing can serve it, and no extraction
|
||||
# can help — `publish` refuses to merge into it, and a job whose sidecar covers the position
|
||||
# skips as already done — so scheduling only kept the player on "preparing" for good. `gen` 0:
|
||||
# there is nothing a <track> could fetch.
|
||||
return {"state": "unavailable", "code": "cache_unreadable", "gen": 0}
|
||||
code = _recent_failure(key)
|
||||
if code and not covered:
|
||||
return {
|
||||
"state": "failed" if code in FINAL_CODES else "unavailable",
|
||||
"code": code,
|
||||
"gen": gen,
|
||||
}
|
||||
# Nothing left that could cover the position for now: the full pass failed, AND so did a window
|
||||
# here. Still not final — the memos lapse, and the rest of the track may be right there.
|
||||
full_failed = _full_failed(rating_key)
|
||||
window_failed = not covered and _window_failed(key, at)
|
||||
if full_failed and window_failed:
|
||||
return {"state": "unavailable", "code": "extract_failed", "gen": gen}
|
||||
if not code:
|
||||
probe, want_window = at, not covered and not window_failed
|
||||
if covered:
|
||||
# Read ahead of an edge instead of waiting to fall off it (see LOOKAHEAD_S). The probe is
|
||||
# just PAST the edge: `covers` is inclusive, so the edge itself reads as already done.
|
||||
edge = max(b for a, b in ranges if a <= at <= b)
|
||||
if edge - at <= LOOKAHEAD_S and edge < _FULL_END_S:
|
||||
probe = edge + 1.0
|
||||
want_window = not _window_failed(key, probe)
|
||||
_schedule(
|
||||
rating_key,
|
||||
sub_ord,
|
||||
probe,
|
||||
want_window=want_window,
|
||||
want_full=not full_failed,
|
||||
viewer=viewer,
|
||||
)
|
||||
# `growing`: whether the track will fill in on its own. False once the full pass has failed and
|
||||
# is backed off — a partial track then stays partial until a window is asked for, so the player
|
||||
# may poll slowly (only to read ahead of an edge) instead of every two seconds for the whole film.
|
||||
return {
|
||||
"state": "partial" if covered else "preparing",
|
||||
"code": None,
|
||||
"gen": gen,
|
||||
"growing": not full_failed,
|
||||
}
|
||||
|
||||
|
||||
def cached_track(rating_key: str, sub_ord: int) -> tuple[str | None, str | None]:
|
||||
"""What the <track> fetch can serve RIGHT NOW, scheduling nothing: the cached VTT (whatever it
|
||||
covers), else the recent failure that explains why there is none.
|
||||
|
||||
Scheduling belongs to `ensure` — the status poll, which knows where the viewer is. The fetch does
|
||||
not: when it scheduled too, it judged readiness at position 0, so a viewer who resumed an hour in
|
||||
got an unrequested window over the head of the file (plus a remount when it landed), and with a
|
||||
failure memo alive it was refused a window that WAS on disk while the poll called it showable."""
|
||||
key = cache_key(rating_key, sub_ord)
|
||||
hit = cache_read(key)
|
||||
return (hit, None) if hit is not None else (None, _recent_failure(key))
|
||||
|
||||
|
||||
def _schedule(
|
||||
rating_key: str,
|
||||
sub_ord: int,
|
||||
at: float,
|
||||
want_window: bool,
|
||||
want_full: bool = True,
|
||||
viewer: int = 0,
|
||||
) -> None:
|
||||
"""Kick off the jobs this track still needs: the window that makes it watchable now, and the one
|
||||
full pass that makes it complete. A 1 s status poll (or a second viewer) joins a job already
|
||||
queued or running instead of starting another ffmpeg.
|
||||
|
||||
A window is joined when one of this track's in-flight windows will COVER the position asked about
|
||||
— decided by what it reads, not by an identity derived from the position. The poll sends the live
|
||||
playhead, so a position-keyed window was a new job every second; bucketing the position instead
|
||||
still queued a second read when the playhead crossed a bucket edge, and joined a backward seek to
|
||||
a window that could not cover it. Nor may it be one window per track: a window cut around the
|
||||
wrong spot (the first poll after a resume can still report 0) made the right one wait for it. The
|
||||
full pass is per ITEM, because one read fans out to every embedded text track of the file."""
|
||||
jobs: list[tuple[tuple, float | None]] = []
|
||||
with _LOCK:
|
||||
if _STOPPING:
|
||||
return
|
||||
n_windows = sum(1 for span in _INFLIGHT.values() if span is not None)
|
||||
n_full = len(_INFLIGHT) - n_windows
|
||||
if want_window and n_windows < _MAX_WINDOWS_INFLIGHT:
|
||||
spans = [
|
||||
span
|
||||
for ident, span in _INFLIGHT.items()
|
||||
if span is not None and ident[1:3] == (rating_key, sub_ord)
|
||||
]
|
||||
if not any(a <= at <= b for a, b in spans) and len(spans) < _MAX_TRACK_WINDOWS:
|
||||
# Rounded to what ffmpeg is told (`-ss %.3f`): the cues come back on ffmpeg's zero,
|
||||
# and shifting them by an unrounded start put them up to half a millisecond off —
|
||||
# enough to spell a timestamp differently from the full pass and double the cue.
|
||||
w_start = float(f"{max(0.0, at - WINDOW_PRE_S):.3f}")
|
||||
ident = ("win", rating_key, sub_ord, w_start)
|
||||
_INFLIGHT[ident] = (w_start, w_start + WINDOW_PRE_S + WINDOW_LEN_S)
|
||||
jobs.append((ident, w_start))
|
||||
mine = sum(1 for owner in _FULL_OWNERS.values() if owner == viewer)
|
||||
if (
|
||||
want_full
|
||||
and n_full < _MAX_FULL_INFLIGHT
|
||||
and ("full", rating_key) not in _INFLIGHT
|
||||
and mine < _MAX_FULL_PER_USER
|
||||
):
|
||||
_INFLIGHT[("full", rating_key)] = None
|
||||
_FULL_OWNERS[rating_key] = viewer
|
||||
jobs.append((("full", rating_key), None))
|
||||
for ident, window_start in jobs:
|
||||
pool = _FULL_POOL if window_start is None else _WIN_POOL
|
||||
pool.submit(_run, ident, rating_key, sub_ord, window_start, at)
|
||||
|
||||
|
||||
def _already_done(key: str, window_start: float | None, at: float) -> bool:
|
||||
"""A queued job looks again before it reads anything. While it waited, another job may have
|
||||
covered it: every embedded track rides each pass over its file, so a window for one language also
|
||||
publishes the others."""
|
||||
if not _exists(_vtt_path(key)):
|
||||
return False
|
||||
ranges, complete = read_meta(key)
|
||||
return complete or (window_start is not None and covers(ranges, at))
|
||||
|
||||
|
||||
def _run(
|
||||
ident: tuple, rating_key: str, sub_ord: int, window_start: float | None, at: float
|
||||
) -> None:
|
||||
key = cache_key(rating_key, sub_ord)
|
||||
try:
|
||||
if _STOPPING or _already_done(key, window_start, at):
|
||||
return
|
||||
_extract_and_publish(rating_key, sub_ord, window_start)
|
||||
except SubtitleUnavailable as e:
|
||||
# The player's own state was the only place this ever showed. `_pass_failed` backs the item
|
||||
# off for an hour and `_fail` records a code, but neither leaves a line, so "my subtitles
|
||||
# never appear" had nothing on the server to read.
|
||||
# A final code is a fact about the file (no such track, a bitmap subtitle); anything else
|
||||
# is this instance failing at something it could do.
|
||||
log.log(
|
||||
logging.INFO if e.code in FINAL_CODES else logging.WARNING,
|
||||
"subtitle extraction gave up",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.pass_failed",
|
||||
rating_key=rating_key,
|
||||
sub_ord=sub_ord,
|
||||
code=e.code,
|
||||
windowed=window_start is not None,
|
||||
),
|
||||
)
|
||||
if e.code == "extract_failed":
|
||||
_pass_failed(rating_key, [key], window_start)
|
||||
else:
|
||||
_fail(key, e.code)
|
||||
except PlexError as e:
|
||||
# No traceback on purpose: a PlexError carries httpx's own message, which names the method,
|
||||
# the URL and the status — the stack under it is this `except` and the client call, and
|
||||
# says nothing the message does not. (The URL holds no token; it travels as a header.)
|
||||
log.warning(
|
||||
"subtitle extraction failed talking to Plex: %s",
|
||||
e,
|
||||
extra=logs.event("plex.subtitles.plex_error", rating_key=rating_key, sub_ord=sub_ord),
|
||||
)
|
||||
_fail(key, "plex_error")
|
||||
except (
|
||||
Exception
|
||||
): # a worker thread has nobody to raise to — record it, let the next poll see it
|
||||
log.exception("subtitle extraction crashed for %s/%s", rating_key, sub_ord)
|
||||
_pass_failed(rating_key, [key], window_start)
|
||||
finally:
|
||||
with _LOCK:
|
||||
_INFLIGHT.pop(ident, None)
|
||||
if ident[0] == "full":
|
||||
_FULL_OWNERS.pop(rating_key, None)
|
||||
|
||||
|
||||
def _pass_failed(
|
||||
rating_key: str,
|
||||
keys: list[str],
|
||||
window_start: float | None,
|
||||
span: tuple[float, float] | None = None,
|
||||
) -> None:
|
||||
"""Record an ffmpeg pass that failed: a full pass backs off the whole item (FULL_RETRY_S); a
|
||||
window marks only the stretch it failed to read, for the tracks it was reading."""
|
||||
if window_start is None:
|
||||
with _LOCK:
|
||||
_sweep_memos()
|
||||
_FULL_FAILED[rating_key] = time.monotonic()
|
||||
return
|
||||
for key in keys:
|
||||
_window_fail(key, span or (window_start, window_start + WINDOW_PRE_S + WINDOW_LEN_S))
|
||||
|
||||
|
||||
def _text_streams(db: Session, rating_key: str) -> tuple[PlexItem, list[dict]]:
|
||||
"""The item plus its subtitle streams, in the ordinal order the API and the UI both use."""
|
||||
it = by_rating_key(db, PlexItem, rating_key)
|
||||
if it is None:
|
||||
raise SubtitleUnavailable("no_item")
|
||||
with PlexClient(db) as plex:
|
||||
part = ((plex.metadata(it.rating_key) or {}).get("Media") or [{}])[0].get("Part") or [{}]
|
||||
return it, [s for s in (part[0].get("Stream") or []) if s.get("streamType") == 3]
|
||||
|
||||
|
||||
def _is_bitmap(stream: dict) -> bool:
|
||||
return (stream.get("codec") or stream.get("format") or "").lower() in BITMAP_SUBS
|
||||
|
||||
|
||||
def _extract_and_publish(rating_key: str, sub_ord: int, window_start: float | None) -> None:
|
||||
"""One extraction job. A sidecar track is fetched whole from Plex (cheap, no ffmpeg); an embedded
|
||||
one is extracted together with EVERY other embedded text track in the file, because they all ride
|
||||
on the same single read — the whole point of the measurement at the top of this module."""
|
||||
with SessionLocal() as db:
|
||||
it, subs = _text_streams(db, rating_key)
|
||||
if sub_ord < 0 or sub_ord >= len(subs):
|
||||
raise SubtitleUnavailable("no_track")
|
||||
want = subs[sub_ord]
|
||||
if _is_bitmap(want):
|
||||
raise SubtitleUnavailable("bitmap")
|
||||
if want.get("key"):
|
||||
# An external sidecar is one small fetch from Plex, never a read of the media file, so a
|
||||
# window job fetches the whole thing too. It must not be left to the full job: that
|
||||
# identity is per ITEM (the embedded fan-out shares it), so a sidecar picked while an
|
||||
# embedded track's full pass was running sat behind that pass for minutes.
|
||||
vtt = _sidecar_vtt(db, want)
|
||||
if not publish(cache_key(rating_key, sub_ord), vtt, (0.0, _FULL_END_S), True):
|
||||
raise SubtitleUnavailable("extract_failed")
|
||||
return
|
||||
if want.get("index") is None:
|
||||
raise SubtitleUnavailable("not_fetchable")
|
||||
src = plex_paths.local_media_path(db, it.file_path)
|
||||
if src is None:
|
||||
raise SubtitleUnavailable("no_file")
|
||||
# Ordinal among the file's EMBEDDED subtitle streams — what `-map 0:s:N` counts. Sidecars sit
|
||||
# in the same list but not in the file, so they must not shift the mapping.
|
||||
targets = {
|
||||
sum(1 for x in subs[:i] if x.get("index") is not None): i
|
||||
for i, s in enumerate(subs)
|
||||
if s.get("index") is not None and not _is_bitmap(s)
|
||||
}
|
||||
# A full pass starts where every track's leading coverage ends: a pass that ran out of time
|
||||
# (`EXTRACT_TIMEOUT_S`) kept what it reached, and restarting from 0 each `FULL_RETRY_S` meant a file
|
||||
# too big to read in one go never completed. Windows handle an offset start already; so does this.
|
||||
start = window_start if window_start is not None else _full_pass_start(rating_key, targets)
|
||||
length = (WINDOW_LEN_S + WINDOW_PRE_S) if window_start is not None else None
|
||||
end = _FULL_END_S if length is None else start + length
|
||||
texts, clean = _ffmpeg_extract(str(src), sorted(targets), start or None, length)
|
||||
for emb_ord, vtt in texts.items():
|
||||
key = cache_key(rating_key, targets[emb_ord])
|
||||
# A pass that exited non-zero holds only what ffmpeg reached before it died. Claim no more
|
||||
# than its last cue, never call it complete, and remember the failure: published as complete,
|
||||
# the track read "ready" for good while its subtitles stopped wherever ffmpeg did (the cache
|
||||
# has no eviction, so nothing would ever have extracted it again).
|
||||
covered_end = end if clean else start + _last_cue_end(vtt)
|
||||
landed = publish(
|
||||
key,
|
||||
shift_vtt(vtt, -start), # zero-based window clock → absolute cue times
|
||||
(start, covered_end),
|
||||
clean and length is None,
|
||||
)
|
||||
if not landed:
|
||||
# A read that could not be kept is a failed read (see publish) — backed off like one.
|
||||
_pass_failed(rating_key, [key], window_start, (start, end))
|
||||
elif not clean:
|
||||
_pass_failed(rating_key, [key], window_start, (covered_end, end))
|
||||
|
||||
|
||||
def _full_pass_start(rating_key: str, targets: dict[int, int]) -> float:
|
||||
"""Where the full pass may begin: the end of the span from 0 that EVERY target track already has
|
||||
(one read feeds them all, so the least-covered track decides), backed off by `WINDOW_PRE_S` so a
|
||||
cue running across the join is read whole — `merge_vtt` folds the overlap. 0 when any track has
|
||||
nothing from the start."""
|
||||
ends = []
|
||||
for ord_ in targets.values():
|
||||
key = cache_key(rating_key, ord_)
|
||||
ranges, _ = read_meta(key) if _exists(_vtt_path(key)) else ([], False)
|
||||
lead = [b for a, b in ranges if a <= 0.0]
|
||||
if not lead:
|
||||
return 0.0
|
||||
ends.append(max(lead))
|
||||
if not ends:
|
||||
return 0.0
|
||||
return max(0.0, float(f"{min(ends) - WINDOW_PRE_S:.3f}"))
|
||||
|
||||
|
||||
def _sidecar_vtt(db: Session, stream: dict) -> str:
|
||||
"""An external subtitle file, fetched through Plex and converted to WebVTT."""
|
||||
with PlexClient(db) as plex:
|
||||
raw = plex.raw_get(stream["key"])
|
||||
codec = (stream.get("codec") or stream.get("format") or "").lower()
|
||||
if codec in ("srt", "subrip", ""):
|
||||
return srt_to_vtt(raw.decode("utf-8", "replace"))
|
||||
with tempfile.NamedTemporaryFile(suffix="." + (codec or "sub"), delete=False) as fh:
|
||||
fh.write(raw)
|
||||
tmp = fh.name
|
||||
try:
|
||||
return ffmpeg_to_vtt(tmp, None)
|
||||
finally:
|
||||
Path(tmp).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _ffmpeg_extract(
|
||||
src: str, emb_ords: list[int], start: float | None, length: float | None
|
||||
) -> tuple[dict[int, str], bool]:
|
||||
"""Extract several embedded subtitle streams to WebVTT in ONE ffmpeg run (= one read of the file).
|
||||
Returns the texts and whether ffmpeg exited cleanly — a non-zero exit can still leave partial
|
||||
output, which is worth publishing but must never be taken for the whole track.
|
||||
|
||||
`start`/`length` bound the read to a window. Without `-copyts` the cues come out on a zero-based
|
||||
clock (the caller shifts them back) — which is exactly why `-copyts` is absent: with it ffmpeg
|
||||
ignores `-t` and reads the whole container anyway (measured)."""
|
||||
if not emb_ords:
|
||||
return {}, True
|
||||
with tempfile.TemporaryDirectory(prefix="plexsub-") as tmpdir:
|
||||
cmd = ["ffmpeg", "-nostdin", "-loglevel", "error"]
|
||||
if start:
|
||||
cmd += ["-ss", f"{start:.3f}"]
|
||||
cmd += ["-i", src]
|
||||
outs = {o: Path(tmpdir) / f"{o}.vtt" for o in emb_ords}
|
||||
for emb_ord, out in outs.items():
|
||||
cmd += ["-map", f"0:s:{emb_ord}"]
|
||||
if length is not None:
|
||||
cmd += ["-t", f"{length:.3f}"]
|
||||
if start:
|
||||
# The input seek lands on the keyframe at or BEFORE `start`, so a cue already running
|
||||
# at `start` comes out at a negative time — and by default the muxer then shifts the
|
||||
# WHOLE output so that cue sits at 0: every cue of the read late by the distance to
|
||||
# that keyframe (1–5 s measured on a real library file, a whole GOP on a synthetic
|
||||
# one), and doubled for good once the full pass merged in beside it. Kept negative, the
|
||||
# times are exact on the zero-based clock (`_vtt_ts_to_s` reads the sign).
|
||||
cmd += ["-avoid_negative_ts", "disabled"]
|
||||
cmd += ["-f", "webvtt", "-y", str(out)]
|
||||
# Popen rather than run(): the child is registered so `shutdown` can kill it — and one that
|
||||
# would start after the shutdown's kill sweep never does, or is killed right here.
|
||||
with _LOCK:
|
||||
if _STOPPING:
|
||||
raise SubtitleUnavailable("extract_failed")
|
||||
proc = subprocess.Popen(
|
||||
cmd, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE
|
||||
)
|
||||
with _LOCK:
|
||||
_PROCS.add(proc)
|
||||
if _STOPPING:
|
||||
proc.kill()
|
||||
timed_out = False
|
||||
try:
|
||||
try:
|
||||
_, err = proc.communicate(timeout=EXTRACT_TIMEOUT_S)
|
||||
except subprocess.TimeoutExpired:
|
||||
# A pass that ran out of time is a pass that died partway: what it reached is kept and
|
||||
# published as partial (the caller's unclean path), like any other non-zero exit. It
|
||||
# used to be killed and its outputs deleted unread — and with the full-pass backoff, the
|
||||
# same doomed read came back every hour. TERM first: ffmpeg closes its outputs on it,
|
||||
# whereas a KILL leaves only what was already flushed in whole blocks (measured: none).
|
||||
timed_out = True
|
||||
proc.terminate()
|
||||
try:
|
||||
_, err = proc.communicate(timeout=_TERM_GRACE_S)
|
||||
except subprocess.TimeoutExpired:
|
||||
# TERM is what makes a timed-out pass worth keeping: ffmpeg closes its outputs
|
||||
# on it. An ffmpeg that ignores TERM leaves only whole blocks already flushed,
|
||||
# so the partial track is short for a reason the later line cannot tell apart.
|
||||
log.info(
|
||||
"the subtitle extraction ignored TERM and was killed",
|
||||
extra=logs.event(
|
||||
"plex.subtitles.extract_killed_after_term",
|
||||
windowed=start is not None,
|
||||
),
|
||||
)
|
||||
proc.kill()
|
||||
_, err = proc.communicate()
|
||||
finally:
|
||||
with _LOCK:
|
||||
_PROCS.discard(proc)
|
||||
got = {
|
||||
o: p.read_text(encoding="utf-8", errors="replace")
|
||||
for o, p in outs.items()
|
||||
if p.exists()
|
||||
}
|
||||
if not got:
|
||||
raise SubtitleUnavailable("extract_failed")
|
||||
clean = proc.returncode == 0 and not timed_out
|
||||
if not clean:
|
||||
log.warning(
|
||||
"ffmpeg subtitle pass %s: %s",
|
||||
"timed out" if timed_out else f"exited {proc.returncode}",
|
||||
(err or b"")[:200],
|
||||
)
|
||||
return got, clean
|
||||
|
||||
+151
-12
@@ -15,20 +15,18 @@ dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are
|
||||
item-detail/player path (P2), not here.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||
from app.plex import detail, people
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Codecs/containers a browser can play in a native <video> without any transcoding.
|
||||
_BROWSER_VIDEO = {"h264"}
|
||||
_BROWSER_AUDIO = {"aac", "mp3", "opus", "vorbis"}
|
||||
@@ -58,9 +56,65 @@ def classify_playable(vcodec: str | None, acodec: str | None, container: str | N
|
||||
return "direct"
|
||||
|
||||
|
||||
# Video codecs the prod iGPU (UHD 630 / Quick Sync Gen 9.5) can hardware-DECODE — the precondition for
|
||||
# the full-GPU VAAPI transcode pipeline. THE single source of this set: plex.stream imports it here so
|
||||
# the runtime encoder choice (`_want_hw`) and this catalog facet can never disagree by codec. A source
|
||||
# outside it (AV1, MPEG-4 ASP/XviD…) is software-only even on the iGPU (it can't decode it to feed its
|
||||
# own encoder).
|
||||
HW_DECODABLE_CODECS = frozenset({"h264", "hevc", "h265", "vp9", "vc1", "mpeg2video"})
|
||||
# Color-transfer values marking an HDR source (PQ / HLG). A 10-bit/HDR source breaks the VAAPI
|
||||
# scale=nv12 path on Gen 9.5 (see plex.stream) → software transcode, so it counts as NOT HW-capable.
|
||||
_HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"})
|
||||
|
||||
|
||||
def transcode_hw_capable(
|
||||
playable: str | None,
|
||||
vcodec: str | None,
|
||||
bit_depth: int | None = None,
|
||||
hdr: bool = False,
|
||||
) -> bool | None:
|
||||
"""The stored `PlexItem.transcode_hw` facet: for a `playable="transcode"` item, whether the prod
|
||||
iGPU can HARDWARE-transcode it (else software libx264 only); None for a non-transcode item. HW needs
|
||||
a HW-decodable codec AND an 8-bit SDR source — a 10-bit/HDR source (bit_depth>8 or an HDR transfer)
|
||||
breaks the VAAPI nv12 pipeline and falls back to software. `bit_depth`/`hdr` are unknown at sync
|
||||
time (the cheap listing carries no Stream), so they default to 8-bit SDR (codec-only) and the
|
||||
bit-depth backfill refines a 10-bit/HDR HEVC to False later."""
|
||||
if playable != "transcode":
|
||||
return None
|
||||
if (vcodec or "").lower() not in HW_DECODABLE_CODECS:
|
||||
return False
|
||||
if bit_depth is not None and bit_depth > 8:
|
||||
return False
|
||||
return not hdr
|
||||
|
||||
|
||||
def _video_stream_props(meta: dict) -> tuple[int | None, bool]:
|
||||
"""(bit_depth, is_hdr) from a FULL Plex metadata dict's video Stream (Media→Part→Stream, streamType
|
||||
1). Only the rich metadata call carries Stream; the cheap listing does not — so this is read by the
|
||||
bit-depth backfill, not the listing sync. (None, False) when absent/unparseable."""
|
||||
for m in meta.get("Media") or []:
|
||||
for part in m.get("Part") or []:
|
||||
for s in part.get("Stream") or []:
|
||||
if s.get("streamType") != 1: # 1 = video
|
||||
continue
|
||||
bd = s.get("bitDepth")
|
||||
try:
|
||||
depth = int(bd) if bd else None
|
||||
# silent-ok: unknown bit depth, which is also what most listings report. The
|
||||
# playback class then falls back to the codec alone, and per-stream noise over a
|
||||
# whole library would drown the sync's own lines.
|
||||
except (TypeError, ValueError):
|
||||
depth = None
|
||||
hdr = str(s.get("colorTrc") or "").lower() in _HDR_TRANSFERS
|
||||
return depth, hdr
|
||||
return None, False
|
||||
|
||||
|
||||
def _epoch(v) -> datetime | None:
|
||||
try:
|
||||
return datetime.fromtimestamp(int(v), tz=timezone.utc) if v else None
|
||||
# silent-ok: a field Plex leaves out of most listing rows. None is a legitimate stored value,
|
||||
# and every caller treats it as "not known" rather than as an error.
|
||||
except (TypeError, ValueError, OSError):
|
||||
return None
|
||||
|
||||
@@ -69,6 +123,7 @@ def _date(v):
|
||||
"""Parse a Plex 'YYYY-MM-DD' string (originallyAvailableAt) → date, or None."""
|
||||
try:
|
||||
return datetime.strptime(v, "%Y-%m-%d").date() if v else None
|
||||
# silent-ok: as `_epoch` above — an absent or unparsable air date is stored as no air date.
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@@ -84,6 +139,7 @@ def _tags(meta: dict, key: str, limit: int | None = None) -> list | None:
|
||||
def _rating(meta: dict) -> float | None:
|
||||
try:
|
||||
return float(meta.get("audienceRating") or meta.get("rating"))
|
||||
# silent-ok: TypeError on `float(None)` is the ordinary path — most items carry no rating.
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@@ -215,8 +271,19 @@ def sync(db: Session) -> dict:
|
||||
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
||||
try:
|
||||
stats["people_enriched"] = _enrich_people_batch(db, plex)
|
||||
except Exception as e: # best-effort — never fail the sync over the backfill
|
||||
log.warning("Plex people enrich batch failed: %s", e)
|
||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
||||
# With the traceback: a blind catch over a batch that fetches AND writes can be
|
||||
# a Plex shape change, a DB error or our own bug, and `%s` told them apart badly.
|
||||
log.exception("the Plex people-enrich batch failed")
|
||||
# Probe a bounded batch of HW-decodable transcode items for bit-depth/HDR, so the
|
||||
# playback-class filter's HW/SW split is precise (a 10-bit/HDR HEVC → software) over
|
||||
# successive runs. Only HW-decodable-codec transcodes are probed — the rest are SW by codec.
|
||||
try:
|
||||
stats["media_probed"] = _probe_media_batch(db, plex)
|
||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
||||
log.exception("the Plex media-probe batch failed")
|
||||
# silent-ok: the return value IS the report. `{"skipped": …}` goes to the scheduler, which
|
||||
# logs the job result, and to the admin route, which answers it to the caller.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
@@ -227,8 +294,8 @@ def sync(db: Session) -> dict:
|
||||
# appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata.
|
||||
try:
|
||||
stats["people_pruned"] = people.prune_orphan_people(db)
|
||||
except Exception as e: # best-effort housekeeping — never fail the sync over it
|
||||
log.warning("Plex people prune failed: %s", e)
|
||||
except Exception: # noqa: BLE001 — best-effort housekeeping; never fail the sync over it
|
||||
log.exception("the Plex people prune failed")
|
||||
log.info("Plex sync done: %s", stats)
|
||||
return stats
|
||||
|
||||
@@ -236,6 +303,51 @@ def sync(db: Session) -> dict:
|
||||
# How many titles to rich-fetch per sync run for the cast backfill — bounds the extra Plex load; the
|
||||
# whole library is covered over successive runs (each fetched item is stamped people_enriched_at once).
|
||||
_ENRICH_BATCH = 60
|
||||
# Same bounded-per-run pattern for the bit-depth probe. Only HW-decodable-codec transcode items are
|
||||
# probed (typically a small minority — the bulk of transcodes are old codecs that are SW by codec), so
|
||||
# a modest batch covers the work-list in a run or two; a stuck item retries next run (watermark unset).
|
||||
_PROBE_BATCH = 60
|
||||
|
||||
|
||||
def _probe_media_batch(db: Session, plex: PlexClient, limit: int = _PROBE_BATCH) -> int:
|
||||
"""Rich-fetch up to `limit` not-yet-probed HW-decodable transcode items, read the video Stream's
|
||||
bit-depth + HDR, and store `video_bit_depth` + the refined `transcode_hw` (a 10-bit/HDR HEVC →
|
||||
False/software). Stamps `media_probed_at` on every attempt (success OR a Plex miss with no Stream),
|
||||
so an item is probed once and not re-fetched every sync; a transient fetch FAILURE leaves the
|
||||
watermark unset to retry. Oldest-id first. Returns how many items were probed this run."""
|
||||
rows = (
|
||||
db.execute(
|
||||
select(PlexItem)
|
||||
.where(
|
||||
PlexItem.playable == "transcode",
|
||||
func.lower(PlexItem.codec_video).in_(HW_DECODABLE_CODECS),
|
||||
PlexItem.media_probed_at.is_(None),
|
||||
)
|
||||
.order_by(PlexItem.id)
|
||||
.limit(limit)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
now = datetime.now(timezone.utc)
|
||||
n = 0
|
||||
for row in rows:
|
||||
try:
|
||||
meta = plex.metadata(row.rating_key) or {}
|
||||
# No traceback and only debug: this is per item over a whole library, and the stack is
|
||||
# the same `plex.metadata` call every time. The retry next run is the handling.
|
||||
except Exception as e: # noqa: BLE001 — transient; leave media_probed_at NULL to retry
|
||||
log.debug("Plex media probe fetch failed for %s: %s", row.rating_key, e)
|
||||
continue
|
||||
bit_depth, hdr = _video_stream_props(meta)
|
||||
row.video_bit_depth = bit_depth
|
||||
row.transcode_hw = transcode_hw_capable("transcode", row.codec_video, bit_depth, hdr)
|
||||
row.media_probed_at = (
|
||||
now # probed (even if Plex carried no Stream — don't re-fetch forever)
|
||||
)
|
||||
n += 1
|
||||
db.commit()
|
||||
return n
|
||||
|
||||
|
||||
def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BATCH) -> int:
|
||||
@@ -271,7 +383,8 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
|
||||
for row in [*movies, *shows]:
|
||||
try:
|
||||
meta = plex.metadata(row.rating_key) or {}
|
||||
except Exception as e:
|
||||
# As in `_probe_media_batch` above: per item, so the message without the stack.
|
||||
except Exception as e: # noqa: BLE001 — transient; leave people_enriched_at NULL
|
||||
log.debug("Plex people enrich fetch failed for %s: %s", row.rating_key, e)
|
||||
continue # leave people_enriched_at NULL so it's retried on a later run
|
||||
people.enrich_row_people(db, row, people.cast_from_meta(meta))
|
||||
@@ -369,15 +482,41 @@ def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
|
||||
row.added_at = _epoch(meta.get("addedAt"))
|
||||
_apply_facet_fields(row, meta) # filterable/orderable extras (shared with shows)
|
||||
mf = _media_facts(meta)
|
||||
new_vcodec = mf.get("codec_video")
|
||||
new_part_key = mf.get("part_key")
|
||||
# The physical file was replaced when its video codec changes OR — for a SINGLE-version item — when its
|
||||
# Plex Part key changes (a re-encode / quality upgrade gets a new part id). Either way any probed
|
||||
# bit-depth is stale, so reset the probe watermark and re-derive transcode_hw from codec alone until the
|
||||
# backfill re-probes. Codec alone is NOT enough: a same-codec bit-depth re-encode (8-bit→10-bit HEVC)
|
||||
# keeps the codec `hevc` but flips the HW class, and only the part-key move catches it.
|
||||
#
|
||||
# The part-key check is gated on single-version because `_media_facts` reads media[0]/part[0], and for a
|
||||
# MULTI-version item (an original + a Plex "Optimized" copy) that order isn't guaranteed stable across
|
||||
# listings — an unguarded part-key compare would then reset+re-probe every sync and oscillate the
|
||||
# transcode_hw bucket between the two versions. A multi-version same-codec re-encode not self-healing is
|
||||
# an acceptable rare edge; a per-sync re-probe loop is not.
|
||||
single_version = len(meta.get("Media") or []) <= 1
|
||||
codec_changed = (row.codec_video or "").lower() != (new_vcodec or "").lower()
|
||||
if codec_changed or (single_version and row.part_key != new_part_key):
|
||||
row.video_bit_depth = None
|
||||
row.media_probed_at = None
|
||||
row.file_path = mf.get("file_path")
|
||||
row.part_key = mf.get("part_key")
|
||||
row.part_key = new_part_key
|
||||
row.container = mf.get("container")
|
||||
row.codec_video = mf.get("codec_video")
|
||||
row.codec_video = new_vcodec
|
||||
row.codec_audio = mf.get("codec_audio")
|
||||
row.size_bytes = mf.get("size_bytes")
|
||||
row.playable = classify_playable(
|
||||
mf.get("codec_video"), mf.get("codec_audio"), mf.get("container")
|
||||
)
|
||||
# transcode_hw: keep the bit-depth backfill's authoritative value (it folds in bit-depth + HDR) once
|
||||
# probed; otherwise (re)derive the provisional codec-only class. A non-transcode item is always None,
|
||||
# even if it had been probed as a transcode before (an audio/container flip to remux without a codec
|
||||
# change wouldn't reset the watermark, so pin it here).
|
||||
if row.playable != "transcode":
|
||||
row.transcode_hw = None
|
||||
elif row.media_probed_at is None:
|
||||
row.transcode_hw = transcode_hw_capable("transcode", row.codec_video, row.video_bit_depth)
|
||||
|
||||
|
||||
def _prune_stale(
|
||||
|
||||
@@ -10,7 +10,6 @@ One-parser rule (mirrors app/plex/detail.py): `parse_tmdb` is the pure parser, `
|
||||
the row. The enrich batch + scheduler entry live here too so the whole TMDB integration is one module.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -19,12 +18,11 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import PlexItem, PlexPerson, PlexShow
|
||||
|
||||
log = logging.getLogger("siftlode.tmdb")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Politeness throttle: TMDB tolerates ~50 req/s; ~33/s leaves comfortable headroom for a shared host.
|
||||
_MIN_INTERVAL = 0.03
|
||||
|
||||
@@ -142,6 +140,12 @@ def guid_to_tmdb_id(guid: str | None, media_type: str, tmdb: TmdbClient) -> int
|
||||
try:
|
||||
return int(ext)
|
||||
except ValueError:
|
||||
# The caller stamps the watermark on a None, so this title is never looked at again:
|
||||
# one unreadable guid costs it its artwork and its cast permanently.
|
||||
log.info(
|
||||
"a Plex guid named TMDB but did not carry a numeric id",
|
||||
extra=logs.event("plex.tmdb.guid_unparsable", provider=provider),
|
||||
)
|
||||
return None
|
||||
source = {"imdb": "imdb_id", "tvdb": "tvdb_id"}.get(provider)
|
||||
if not source:
|
||||
@@ -499,7 +503,17 @@ def enrich_person(db: Session, person: PlexPerson) -> bool:
|
||||
# doesn't tie up a threadpool worker for the default 30s.
|
||||
with TmdbClient(db, timeout=8.0) as tmdb:
|
||||
payload = tmdb.fetch_person(person.tmdb_person_id)
|
||||
except TmdbError:
|
||||
except TmdbError as exc:
|
||||
# False leaves the person un-enriched and unstamped, so it is retried — but the page in
|
||||
# front of the user right now shows an empty biography with nothing to say why.
|
||||
log.warning(
|
||||
"the TMDB person lookup failed",
|
||||
extra=logs.event(
|
||||
"plex.tmdb.person_enrich_failed",
|
||||
person_id=person.id,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
return False
|
||||
if payload is None:
|
||||
person.tmdb_enriched_at = datetime.now(timezone.utc)
|
||||
|
||||
+131
-28
@@ -11,17 +11,23 @@ already pages) and upsert them into the owner's `plex_states`. Plex wins on over
|
||||
exist only in Siftlode are left untouched (union, Plex-authoritative on the intersection).
|
||||
|
||||
Phases B (Siftlode→Plex push via scrobble/timeline) and C (incremental Plex→Siftlode via the watch
|
||||
history + onDeck, last-write-wins) build on this and land in later ships.
|
||||
history + onDeck) build on this and land in later ships. As of S2 Plex is the source of truth: the
|
||||
player pushes the resume position back to Plex on the periodic checkpoint (not just the final flush),
|
||||
and the incremental pull resolves a genuine divergence to Plex's state (see `_pull_apply`) rather than
|
||||
running a last-write-wins comparison.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
# On the PlexError lines in this module: none of them carries a traceback, deliberately. A
|
||||
# PlexError is httpx's own message — the method, the URL and the status of one call — and the stack
|
||||
# beneath it is always this `except` plus `PlexClient._get`. The attrs beside the line say which
|
||||
# user and which phase, which is the part the message cannot know.
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import logs, sysconfig
|
||||
from app.db import SessionLocal
|
||||
from app.models import PlexItem, PlexLink, PlexState
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
@@ -33,8 +39,7 @@ from app.plex.sync import (
|
||||
_paginate,
|
||||
)
|
||||
|
||||
log = logging.getLogger("siftlode.plex.watch")
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# A resume position below this many seconds isn't worth mirroring — matches the player's own
|
||||
# _PROGRESS_MIN_S (a stray few-second seek is not "in progress").
|
||||
_PROGRESS_MIN_S = 5
|
||||
@@ -145,6 +150,8 @@ def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
||||
link.initial_import_done = True
|
||||
link.last_watch_sync_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
# silent-ok: the `{"skipped": …}` return is the report — the admin route answers it and the
|
||||
# scheduler logs it on the job line.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
@@ -293,12 +300,6 @@ def push_bulk_state_to_plex(user_id: int, changes: list[tuple[int, str, str]]) -
|
||||
|
||||
# --- Phase C: incremental + full Plex ↔ Siftlode reconcile (scheduler jobs) ------------------------
|
||||
|
||||
# Clock-skew / round-trip slack when comparing a Plex timestamp to a Siftlode one. A state we just
|
||||
# pushed (Plex stamps viewedAt a beat after our watched_at) must not read as "newer on Plex" and get
|
||||
# pulled back with a different shape. Genuine Plex-side edits are minutes from our writes — far
|
||||
# outside this — so the window only absorbs skew, never real changes.
|
||||
_LWW_TOLERANCE_S = 90
|
||||
|
||||
|
||||
def _active_sync_links(db: Session) -> list[PlexLink]:
|
||||
"""Every link with two-way sync actually running (owner MVP = one row). Iterated uniformly so
|
||||
@@ -329,8 +330,18 @@ def _resolve_account_id(db: Session, plex: PlexClient, link: PlexLink) -> int:
|
||||
if int(a.get("id") or -1) == 1:
|
||||
username = a.get("name")
|
||||
break
|
||||
except PlexError:
|
||||
pass
|
||||
except PlexError as exc:
|
||||
# The fallback account id 1 is right for the admin-token setup, so sync keeps working —
|
||||
# but the username is never stored, and the UI then has no name to show for the link,
|
||||
# exactly as if the server had not offered one.
|
||||
log.info(
|
||||
"the Plex account list could not be read; the link keeps the default account",
|
||||
extra=logs.event(
|
||||
"plex.watch.accounts_probe_failed",
|
||||
user_id=link.user_id,
|
||||
reason=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
link.plex_account_id = acct_id
|
||||
if username:
|
||||
link.plex_username = username
|
||||
@@ -338,11 +349,6 @@ def _resolve_account_id(db: Session, plex: PlexClient, link: PlexLink) -> int:
|
||||
return acct_id
|
||||
|
||||
|
||||
def _sift_ts(st: PlexState) -> datetime | None:
|
||||
"""The freshest Siftlode-side timestamp on a row, for last-write-wins."""
|
||||
return max([t for t in (st.watched_at, st.progress_updated_at) if t], default=None)
|
||||
|
||||
|
||||
def _same_state(st: PlexState | None, status: str, pos: int) -> bool:
|
||||
"""Whether a row already matches a target (status, pos) — the core ping-pong guard: when nothing
|
||||
would change we skip the write entirely (no row churn, no timestamp bump, no re-pull)."""
|
||||
@@ -370,9 +376,20 @@ def _pull_apply(
|
||||
if st is not None and not st.synced_to_plex:
|
||||
st.synced_to_plex = True # already matches Plex → just settle the flag, no churn
|
||||
return None
|
||||
# Conflict: keep Siftlode when it's newer than Plex (plus skew slack).
|
||||
sts = _sift_ts(st) if st else None
|
||||
if sts and plex_ts and plex_ts <= sts + timedelta(seconds=_LWW_TOLERANCE_S):
|
||||
# Plex-authoritative (S2): on a genuine divergence for an item Plex knows about, Plex wins — no
|
||||
# last-write-wins tug-of-war. This is safe because prompt write-back (playback.py pushes the resume
|
||||
# position to Plex on the ~10s checkpoint, not just on the final flush) keeps Plex current, so a
|
||||
# local edit that's "newer" than Plex has almost always already been pushed and matched above by
|
||||
# `_same_state`; a row that reaches here really is out of step with the source of truth. The
|
||||
# `_same_state` + `synced_to_plex` short-circuit above still absorbs the freshly-pushed case (Plex
|
||||
# echoes the same shape a beat later) so a just-pushed row never bounces.
|
||||
if st is not None and not st.synced_to_plex:
|
||||
# Durability guard: this row holds a local change that never reached Plex — its immediate push
|
||||
# failed, so it's still dirty. Do NOT let the Plex-authoritative pull overwrite it and clear the
|
||||
# flag: that would strand the local value forever, since `_repush_dirty` only retries dirty rows.
|
||||
# Leave it dirty; `_repush_dirty` (later in this same run) pushes the local value up to Plex. This
|
||||
# closes the "Plex flapped mid-session" lost-update — the one case the Plex-wins comment's
|
||||
# "almost always already pushed" assumption above excludes.
|
||||
return None
|
||||
if st is None:
|
||||
st = PlexState(user_id=user_id, item_id=item_id)
|
||||
@@ -436,14 +453,89 @@ def _repush_dirty(db: Session, plex: PlexClient, link: PlexLink) -> int:
|
||||
return n
|
||||
|
||||
|
||||
# How many stale in-progress movies we probe per incremental run (one metadata() call each). The set
|
||||
# is normally tiny (movies started but no longer on Plex's Continue-Watching); the daily full reconcile
|
||||
# is the backstop for anything beyond the cap. Most-recently-touched are probed first.
|
||||
_MOVIE_PROBE_CAP = 100
|
||||
|
||||
|
||||
def _reconcile_inprogress_movies(
|
||||
db: Session, plex: PlexClient, link: PlexLink, on_deck_ids: set[int]
|
||||
) -> dict[str, int]:
|
||||
"""Close the movie "stuck in progress" gap that the history + onDeck feeds structurally miss.
|
||||
|
||||
When a movie is finished on Plex (often by scrubbing through the end credits) Plex sets
|
||||
``viewCount>0``, clears ``viewOffset`` and DROPS the movie from Continue-Watching. So onDeck no
|
||||
longer lists it (and onDeck can only ever ADD an in-progress row, never resolve one to watched —
|
||||
see `_pull_apply`), and a scrub-to-end frequently emits no `/status/sessions/history` session row,
|
||||
so the history feed misses it too. The stale local ``status='new', position>0`` row then survives
|
||||
until the *daily* full reconcile. Movies are hit hardest: a movie is a single leaf with no
|
||||
episode-rollup to mask the stuck row, and once watched it leaves onDeck entirely.
|
||||
|
||||
Fix: for each local in-progress MOVIE that Plex's onDeck no longer lists, read the item's current
|
||||
per-(token-)account ``viewCount``/``viewOffset`` with one cheap `metadata()` call and apply it via
|
||||
the shared `_pull_apply` — flipping to watched when Plex says so. Returns ``{"watched", "in_progress",
|
||||
"probed"}`` counts. Episodes are intentionally left to onDeck/history (they promote the next episode
|
||||
and aren't affected). Note: like the rest of this job, `metadata()` reads the TOKEN account (the
|
||||
owner) — a HTPC playing under a managed Plex user (accountID>1) is out of scope here as elsewhere."""
|
||||
conds = [
|
||||
PlexState.user_id == link.user_id,
|
||||
PlexState.status.notin_(("watched", "hidden")),
|
||||
PlexState.position_seconds > _PROGRESS_MIN_S,
|
||||
PlexItem.kind == "movie",
|
||||
]
|
||||
if on_deck_ids:
|
||||
conds.append(PlexState.item_id.notin_(on_deck_ids))
|
||||
rows = db.execute(
|
||||
select(PlexState, PlexItem.rating_key)
|
||||
.join(PlexItem, PlexItem.id == PlexState.item_id)
|
||||
.where(*conds)
|
||||
.order_by(PlexState.progress_updated_at.desc().nullslast())
|
||||
.limit(_MOVIE_PROBE_CAP + 1)
|
||||
).all()
|
||||
if len(rows) > _MOVIE_PROBE_CAP:
|
||||
rows = rows[:_MOVIE_PROBE_CAP]
|
||||
log.warning(
|
||||
"in-progress movie probe hit the %d cap (user %s); reconcile catches the tail",
|
||||
_MOVIE_PROBE_CAP,
|
||||
link.user_id,
|
||||
)
|
||||
out = {"watched": 0, "in_progress": 0, "probed": 0}
|
||||
for st, rk in rows:
|
||||
try:
|
||||
meta = plex.metadata(rk)
|
||||
except PlexError as e:
|
||||
log.warning(
|
||||
"movie watch-probe failed (user %s, item %s): %s", link.user_id, st.item_id, e
|
||||
)
|
||||
continue
|
||||
out["probed"] += 1
|
||||
if meta is None:
|
||||
continue
|
||||
res = _plex_watch_to_state(
|
||||
meta.get("viewCount"), meta.get("viewOffset"), meta.get("lastViewedAt")
|
||||
)
|
||||
if res is None:
|
||||
# Plex holds no watch signal now (viewCount 0, no offset) — don't clear it here; an un-watch
|
||||
# is the daily reconcile's job, and a Siftlode-only in-progress row must be preserved.
|
||||
continue
|
||||
status, pos, watched_at, prog_at = res
|
||||
ch = _pull_apply(db, link.user_id, st.item_id, status, pos, watched_at or prog_at)
|
||||
if ch:
|
||||
out[ch] += 1
|
||||
return out
|
||||
|
||||
|
||||
def run_plex_watch_sync(db: Session) -> dict:
|
||||
"""Scheduler job (default 30 min) — the cheap incremental pass. Pulls Plex-side changes since the
|
||||
high-water mark (watch history + on-deck, filtered to the owner account) into Siftlode under
|
||||
last-write-wins, then re-pushes any still-unsynced local states. Does NOT propagate un-watches
|
||||
(the history feed only carries positive views) — that's `run_plex_watch_reconcile`'s job."""
|
||||
"""Scheduler job (default 15 min) — the cheap incremental pass. Pulls Plex-side changes since the
|
||||
high-water mark (watch history + on-deck, filtered to the owner account) into Siftlode, resolving a
|
||||
divergence to Plex (the source of truth — see `_pull_apply`), reconciles stale in-progress movies
|
||||
Plex has since finished (see `_reconcile_inprogress_movies`), then re-pushes any still-unsynced
|
||||
local states. Does NOT propagate un-watches (the history feed only carries positive views) — that's
|
||||
`run_plex_watch_reconcile`'s job."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
return {"skipped": "disabled"}
|
||||
total = {"links": 0, "watched": 0, "in_progress": 0, "repushed": 0}
|
||||
total = {"links": 0, "watched": 0, "in_progress": 0, "repushed": 0, "movie_probes": 0}
|
||||
for link in _active_sync_links(db):
|
||||
total["links"] += 1
|
||||
try:
|
||||
@@ -463,10 +555,14 @@ def run_plex_watch_sync(db: Session) -> dict:
|
||||
if ch:
|
||||
total[ch] += 1
|
||||
|
||||
# onDeck resolves resume-position changes AND tells us which items Plex still considers
|
||||
# in-progress (needed by the movie reconcile below to know which local rows are stale).
|
||||
on_deck_ids: set[int] = set()
|
||||
for row in plex.on_deck():
|
||||
iid = rk_to_id.get(str(row.get("ratingKey") or ""))
|
||||
if iid is None:
|
||||
continue
|
||||
on_deck_ids.add(iid)
|
||||
off = int(row.get("viewOffset") or 0) // 1000
|
||||
if off < _PROGRESS_MIN_S:
|
||||
continue
|
||||
@@ -476,9 +572,15 @@ def run_plex_watch_sync(db: Session) -> dict:
|
||||
if ch:
|
||||
total[ch] += 1
|
||||
|
||||
mv = _reconcile_inprogress_movies(db, plex, link, on_deck_ids)
|
||||
total["watched"] += mv["watched"]
|
||||
total["in_progress"] += mv["in_progress"]
|
||||
total["movie_probes"] += mv["probed"]
|
||||
|
||||
total["repushed"] += _repush_dirty(db, plex, link)
|
||||
link.last_watch_sync_at = now
|
||||
db.commit()
|
||||
# silent-ok: the `{"skipped": …}` return is the report, as above.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
@@ -491,7 +593,7 @@ def run_plex_watch_sync(db: Session) -> dict:
|
||||
def run_plex_watch_reconcile(db: Session) -> dict:
|
||||
"""Scheduler job (default daily) — the authoritative full reconcile. Rescans every section and,
|
||||
per item, uses `synced_to_plex` to settle what the incremental feed can't:
|
||||
- Plex has a state, Siftlode differs → last-write-wins (Plex usually the newer here).
|
||||
- Plex has a state, Siftlode differs → Plex wins (the source of truth — see `_pull_apply`).
|
||||
- Siftlode has a state, Plex has NO record:
|
||||
· synced_to_plex=True → Plex un-watched it since our last sync → clear it (un-watch).
|
||||
· synced_to_plex=False → a local change that never reached Plex → re-push it (via
|
||||
@@ -530,6 +632,7 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
||||
total["repushed"] += _repush_dirty(db, plex, link)
|
||||
link.last_watch_sync_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
# silent-ok: the `{"skipped": …}` return is the report, as above.
|
||||
except PlexNotConfigured as e:
|
||||
return {"skipped": str(e)}
|
||||
except PlexError as e:
|
||||
|
||||
@@ -37,6 +37,7 @@ class QuotaAction:
|
||||
PLAYLISTS_DELETE = "playlists_delete"
|
||||
VIDEOS_BACKFILL_RECENT = "videos_backfill_recent"
|
||||
VIDEOS_BACKFILL_FULL = "videos_backfill_full"
|
||||
VIDEOS_FEED_FALLBACK = "videos_feed_fallback"
|
||||
VIDEOS_ENRICH = "videos_enrich"
|
||||
VIDEOS_LOOKUP = "videos_lookup"
|
||||
VIDEOS_SEARCH = "videos_search"
|
||||
|
||||
+74
-11
@@ -1,44 +1,107 @@
|
||||
"""A tiny in-process sliding-window rate limiter.
|
||||
|
||||
Generic groundwork: keyed by an arbitrary string (e.g. a client IP), so it can throttle any
|
||||
Generic groundwork: keyed by an arbitrary string (e.g. a client — `client_key`), so it can throttle any
|
||||
endpoint. Single-worker uvicorn → in-memory state is sufficient; it resets on restart, which
|
||||
is fine for abuse throttling (not for anything that must survive a deploy). Not shared across
|
||||
processes — if we ever run multiple workers, swap the backing store for Redis behind the same
|
||||
``allow()`` interface.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import threading
|
||||
import time
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
class RateLimiter:
|
||||
def __init__(self, max_events: int, window_seconds: float):
|
||||
"""At most `max_events` attempts per `window_seconds` per key — or, when each attempt passes a
|
||||
`cost`, at most that much total cost (a byte budget counts bytes, not calls)."""
|
||||
|
||||
def __init__(self, max_events: float, window_seconds: float):
|
||||
self.max_events = max_events
|
||||
self.window = window_seconds
|
||||
self._hits: dict[str, list[float]] = {}
|
||||
# key → (monotonic time, cost) of every counted attempt still inside the window
|
||||
self._hits: dict[str, list[tuple[float, float]]] = {}
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def allow(self, key: str) -> bool:
|
||||
"""Record an attempt for ``key`` and return whether it is within the limit.
|
||||
def reset(self) -> None:
|
||||
"""Forget every recorded attempt. Each limiter is a module-level singleton, so a test suite
|
||||
sharing one process would otherwise pool all of its attempts into a single window and
|
||||
eventually trip an unrelated test with a 429."""
|
||||
with self._lock:
|
||||
self._hits.clear()
|
||||
|
||||
True -> under the cap (the attempt is counted).
|
||||
False -> the cap for the current window is already reached (attempt NOT counted, so a
|
||||
def exhausted(self, key: str) -> bool:
|
||||
"""Whether ``key``'s window is already full (``allow`` would refuse it), WITHOUT counting an
|
||||
attempt — for a limit that counts only some outcomes: ask it first, ``allow`` the ones that
|
||||
count."""
|
||||
cutoff = time.monotonic() - self.window
|
||||
with self._lock:
|
||||
return sum(c for t, c in self._hits.get(key, []) if t > cutoff) >= self.max_events
|
||||
|
||||
def allow(self, key: str, cost: float = 1) -> bool:
|
||||
"""Record an attempt costing ``cost`` for ``key`` and return whether it is within the limit.
|
||||
|
||||
True -> within the cap (the attempt is counted).
|
||||
False -> the attempt would pass the cap for the current window (attempt NOT counted, so a
|
||||
blocked caller can't keep pushing the window forward)."""
|
||||
now = time.monotonic()
|
||||
cutoff = now - self.window
|
||||
with self._lock:
|
||||
hits = [t for t in self._hits.get(key, []) if t > cutoff]
|
||||
if len(hits) >= self.max_events:
|
||||
hits = [(t, c) for t, c in self._hits.get(key, []) if t > cutoff]
|
||||
if sum(c for _, c in hits) + cost > self.max_events:
|
||||
self._hits[key] = hits
|
||||
return False
|
||||
hits.append(now)
|
||||
hits.append((now, cost))
|
||||
self._hits[key] = hits
|
||||
# Opportunistic cleanup so abandoned keys don't accumulate unboundedly.
|
||||
if len(self._hits) > 4096:
|
||||
for k in list(self._hits):
|
||||
fresh = [t for t in self._hits[k] if t > cutoff]
|
||||
fresh = [(t, c) for t, c in self._hits[k] if t > cutoff]
|
||||
if fresh:
|
||||
self._hits[k] = fresh
|
||||
else:
|
||||
del self._hits[k]
|
||||
return True
|
||||
|
||||
|
||||
def _norm_ip(s: str) -> ipaddress.IPv4Address | ipaddress.IPv6Address | None:
|
||||
"""Parse an IP for comparison, unwrapping IPv4-mapped IPv6 (`::ffff:10.0.0.1` → `10.0.0.1`) so a
|
||||
plain-IPv4 allowlist still matches a peer that surfaces in mapped form. None for a non-IP string."""
|
||||
try:
|
||||
ip = ipaddress.ip_address(s.strip())
|
||||
# silent-ok: parsing an untrusted header value, where "not an IP" is a documented answer this
|
||||
# function returns — and a line per request is what an attacker would aim for.
|
||||
except ValueError:
|
||||
return None
|
||||
return ip.ipv4_mapped if getattr(ip, "ipv4_mapped", None) else ip
|
||||
|
||||
|
||||
def client_key(request: Request) -> str:
|
||||
"""Best-effort client identity for rate limiting: its IP, or for IPv6 the /64 the address sits in — a
|
||||
subscriber is handed a whole /64, so keyed on the full address one client would have endless identities
|
||||
to spread its attempts over. X-Forwarded-For is client-controlled, so we trust it
|
||||
ONLY when the request actually arrived from a configured reverse proxy (settings.trusted_proxy_ips
|
||||
— the address the proxy connects FROM, e.g. the VPS Caddy's WireGuard peer IP). Our proxy APPENDS
|
||||
the client it saw to XFF, so the RIGHTMOST entry is the real client and can't be forged by a
|
||||
client pre-seeding the header. Any other peer (someone hitting the app port directly, bypassing the
|
||||
proxy) is untrusted — we use its real socket IP, so it can't spoof its rate-limit identity.
|
||||
|
||||
SAFETY: this relies on `request.client` being the TRUE socket peer. uvicorn is started WITHOUT
|
||||
`--proxy-headers` (see entrypoint.sh) precisely so it never rewrites `request.client` from XFF —
|
||||
do NOT add that flag, or the trust check below would compare against a forgeable value."""
|
||||
peer = request.client.host if request.client else "unknown"
|
||||
client = peer
|
||||
trusted = {_norm_ip(p) for p in settings.trusted_proxy_ip_set} - {None}
|
||||
if _norm_ip(peer) in trusted:
|
||||
xff = request.headers.get("x-forwarded-for")
|
||||
parts = [p.strip() for p in (xff or "").split(",") if p.strip()]
|
||||
if parts:
|
||||
client = parts[-1]
|
||||
ip = _norm_ip(client)
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
return str(ipaddress.IPv6Network((ip, 64), strict=False))
|
||||
return client
|
||||
|
||||
+13
-6
@@ -13,12 +13,12 @@ mutated from a single thread.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from starlette.websockets import WebSocket
|
||||
from app import logs
|
||||
|
||||
log = logging.getLogger("siftlode.realtime")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
@@ -42,9 +42,16 @@ class ConnectionManager:
|
||||
for ws in list(self._conns.get(user_id, ())):
|
||||
try:
|
||||
await ws.send_json(payload)
|
||||
except Exception:
|
||||
# Stale socket; it'll be cleaned up by its own disconnect handler.
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001 — one dead socket must not stop the others
|
||||
# Stale socket; it'll be cleaned up by its own disconnect handler. But this is
|
||||
# also where a live update goes missing: the UI simply never changes, and there
|
||||
# is no request behind it whose failure anyone could look up.
|
||||
log.debug(
|
||||
"a live update could not be pushed to a socket",
|
||||
extra=logs.event(
|
||||
"realtime.push_failed", user_id=user_id, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
|
||||
def push(self, user_id: int, payload: dict) -> None:
|
||||
"""Schedule a push onto the WebSocket event loop. Safe to call from a sync route
|
||||
@@ -54,7 +61,7 @@ class ConnectionManager:
|
||||
return
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(self._push(user_id, payload), loop)
|
||||
except Exception:
|
||||
except Exception: # noqa: BLE001 — a closed loop or socket must not fail the caller
|
||||
log.debug("ws push failed", exc_info=True)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""A request the app refuses, said in a way the UI can put in the reader's language.
|
||||
|
||||
Every refusal the API answers carries a structured `detail` — `{code, reason, …params}` — and the client
|
||||
translates it (`lib/api/core.ts` `localizeDetail`). A bare English string reached the Hungarian UI
|
||||
verbatim: the blocking error dialog, and every inline surface that printed `HttpError.detail`, showed
|
||||
whatever sentence the route happened to hand-write. `refused` is the generic family
|
||||
(`errors.refused.<reason>`, the params interpolated into the sentence); the older specific codes
|
||||
(`not_found`, `youtube`, `playlist`, the admin refusals, …) stay, since each is already translated.
|
||||
|
||||
A test (`test_http_details.py`) reads every `HTTPException` in the app and fails on a string detail,
|
||||
and the frontend's contract test requires every `refused(…, "<reason>")` to be named in both
|
||||
languages — so the rule is enforced, not described.
|
||||
"""
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
|
||||
def refused(status: int, reason: str, **params: object) -> HTTPException:
|
||||
"""The refusal to raise: `raise refused(409, "view_name_taken", name=name)`. `params` travel in
|
||||
the detail for the sentence to interpolate (`{{name}}`); keep them to values the reader may see."""
|
||||
return HTTPException(status_code=status, detail={"code": "refused", "reason": reason, **params})
|
||||
|
||||
|
||||
# `sysconfig.InvalidSetting.reason` → the refusal it answers as. A dict of literals, so the frontend's
|
||||
# contract test can read every reason this can produce.
|
||||
_SETTING_REASONS = {
|
||||
"not_a_number": "setting_not_a_number",
|
||||
"too_low": "setting_too_low",
|
||||
"too_high": "setting_too_high",
|
||||
"not_a_choice": "setting_not_a_choice",
|
||||
"no_encryption_key": "no_encryption_key",
|
||||
}
|
||||
|
||||
|
||||
def setting_refused(exc) -> HTTPException:
|
||||
"""A value the admin entered that `sysconfig.set_value` refused (`exc`: its `InvalidSetting`)."""
|
||||
return refused(400, _SETTING_REASONS[exc.reason], key=exc.key, **exc.params)
|
||||
@@ -8,21 +8,32 @@ one existence-leak policy.
|
||||
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi import HTTPException, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import Base
|
||||
from app.models import User
|
||||
from app.issues import gitea
|
||||
from app.refusal import refused
|
||||
from app.youtube.client import YouTubeError, refusal_reason
|
||||
|
||||
M = TypeVar("M", bound=Base)
|
||||
|
||||
|
||||
def get_or_404(db: Session, model: type[M], id_: Any, *, detail: str = "Not found") -> M:
|
||||
def not_found(what: str = "generic") -> HTTPException:
|
||||
"""The 404 for a row that is gone — structured, so the bilingual UI says it in the reader's
|
||||
language (`errors.notFound.<what>`, generic fallback). This is the 404 a user actually meets: the
|
||||
thing was deleted in another tab or on another device. It used to be a bare English string
|
||||
("Unknown view", "Not found"), which the Hungarian UI toasted verbatim."""
|
||||
return HTTPException(status_code=404, detail={"code": "not_found", "reason": what})
|
||||
|
||||
|
||||
def get_or_404(db: Session, model: type[M], id_: Any, *, what: str = "generic") -> M:
|
||||
"""Fetch a row by primary key or raise 404. Replaces the `db.get(Model, id); if None: 404`
|
||||
idiom repeated across routes."""
|
||||
obj = db.get(model, id_)
|
||||
if obj is None:
|
||||
raise HTTPException(status_code=404, detail=detail)
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
|
||||
@@ -33,13 +44,57 @@ def owned_or_404(
|
||||
user: User,
|
||||
*,
|
||||
owner_attr: str = "user_id",
|
||||
detail: str = "Not found",
|
||||
what: str = "generic",
|
||||
) -> M:
|
||||
"""Fetch a row by primary key that `user` owns, or raise 404. A row owned by someone
|
||||
else 404s (NOT 403) on purpose — a 403 would confirm the id exists to a stranger.
|
||||
Generalizes the near-identical per-route ownership helpers (e.g. plex's
|
||||
`_own_playlist_or_404`, the saved-views owner check)."""
|
||||
obj = get_or_404(db, model, id_, detail=detail)
|
||||
It replaced the near-identical per-route ownership helpers, and is called directly — a thin
|
||||
`_own_x` wrapper around it is the ad-hoc shape CLAUDE.md's patterns table rules out."""
|
||||
obj = get_or_404(db, model, id_, what=what)
|
||||
if getattr(obj, owner_attr) != user.id:
|
||||
raise HTTPException(status_code=404, detail=detail)
|
||||
raise not_found(what)
|
||||
return obj
|
||||
|
||||
|
||||
def youtube_refusal(exc: Exception) -> HTTPException:
|
||||
"""The answer to an action YouTube refused: a 422 whose reason the UI translates
|
||||
(`errors.youtube.<reason>`). One definition, because the routes had grown three shapes of it —
|
||||
a bare English string quoting the upstream message, a playlist-scoped reason, and nothing at
|
||||
all (a raw 500 from the manual syncs). The upstream failure is already logged where it
|
||||
happened (`youtube.api.error`). A failure that is not the Data API's (the search scraper's) is
|
||||
YouTube being unavailable to us — the retryable answer."""
|
||||
reason = refusal_reason(exc) if isinstance(exc, YouTubeError) else "unavailable"
|
||||
return HTTPException(status_code=422, detail={"code": "youtube", "reason": reason})
|
||||
|
||||
|
||||
async def capped_body(request: Request, limit: int) -> bytes:
|
||||
"""The request body, refused (413 `too_large`) the moment it passes `limit` bytes while still
|
||||
arriving. Counted, not taken from Content-Length: a declared length can be absent (chunked) or
|
||||
wrong, and counting stops reading at the same point a declared-length check would have."""
|
||||
body = bytearray()
|
||||
async for chunk in request.stream():
|
||||
body += chunk
|
||||
if len(body) > limit:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
return bytes(body)
|
||||
|
||||
|
||||
def issue_refused(exc: gitea.GiteaError) -> HTTPException:
|
||||
"""The refusal an issue-tracker failure (E16) answers as — the send, the preview and the
|
||||
admin's connection test alike. Literal reasons, one per case: the frontend's contract test reads
|
||||
them to require a sentence for each in both languages."""
|
||||
match exc.reason:
|
||||
case "not_configured":
|
||||
return refused(400, "issues_not_configured")
|
||||
case "unreachable":
|
||||
return refused(422, "issues_unreachable")
|
||||
case "bad_token":
|
||||
return refused(422, "issues_bad_token")
|
||||
case "repo_not_found":
|
||||
return refused(422, "issues_repo_not_found")
|
||||
case "public_repo":
|
||||
return refused(422, "issues_public_repo")
|
||||
case "issues_disabled":
|
||||
return refused(422, "issues_disabled")
|
||||
case _:
|
||||
return refused(422, "issues_rejected")
|
||||
|
||||
+490
-51
@@ -1,20 +1,26 @@
|
||||
"""Admin-only onboarding: review access requests (Invites), approve/deny, manual add.
|
||||
Also the demo-account admin surface: the demo email whitelist + a manual state reset."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit
|
||||
from app import email as email_mod
|
||||
from app import mailtext
|
||||
from app.audit import AuditAction
|
||||
from app.auth import (
|
||||
admin_user,
|
||||
can_activate,
|
||||
count_admins,
|
||||
env_allowed_canonicals,
|
||||
find_user_by_email,
|
||||
get_or_create_demo_user,
|
||||
is_allowed,
|
||||
operator_contact,
|
||||
purge_user,
|
||||
)
|
||||
@@ -25,13 +31,48 @@ from app.models import (
|
||||
Invite,
|
||||
User,
|
||||
)
|
||||
from app.routes.auth import revoke_preset_password_if_unverified
|
||||
from app.utils import iso, valid_email
|
||||
from app.routes._common import get_or_404
|
||||
from app.routes.auth import (
|
||||
APPROVAL_VERIFY_TTL,
|
||||
issue_set_password_link,
|
||||
issue_verification_link,
|
||||
send_verification_link,
|
||||
)
|
||||
from app.utils import canonical_email, iso, valid_email
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
|
||||
|
||||
def _serialize(inv: Invite) -> dict:
|
||||
def _accounts_for(db: Session, emails: Iterable[str]) -> set[str]:
|
||||
"""Which of these addresses already have an account, as a set of CANONICAL forms. Batched, so
|
||||
listing the invites costs one extra query. An invite whose address has an account must not be
|
||||
deletable: `is_allowed` reads this row, so removing it would silently revoke a live user's access.
|
||||
|
||||
Canonical on both sides, because `is_allowed` matches that way: an invite for `x.y@gmail.com`
|
||||
whose account registered as `xy@gmail.com` would otherwise look account-less, offer the admin a
|
||||
delete button, and take away a live user's access — precisely the accident this guard exists to
|
||||
prevent."""
|
||||
wanted = {canonical_email(e) for e in emails if e}
|
||||
if not wanted:
|
||||
return set()
|
||||
rows = db.execute(
|
||||
select(User.email_canonical).where(User.email_canonical.in_(wanted))
|
||||
).scalars()
|
||||
return {e for e in rows if e}
|
||||
|
||||
|
||||
def _env_allowed(email: str) -> bool:
|
||||
"""Whether the env bootstrap lists still let this address in. `is_allowed` consults them only
|
||||
when no Invite row exists, so this is exactly what deleting the row would fall back to — and the
|
||||
admin has to know that BEFORE they confirm, not from a toast afterwards.
|
||||
|
||||
Canonical, because `is_allowed` is: a warning that disagrees with the gate it is warning about is
|
||||
worse than no warning."""
|
||||
return canonical_email(email or "") in env_allowed_canonicals()
|
||||
|
||||
|
||||
def _serialize(inv: Invite, *, has_account: bool = False) -> dict:
|
||||
return {
|
||||
"id": inv.id,
|
||||
"email": inv.email,
|
||||
@@ -40,6 +81,19 @@ def _serialize(inv: Invite) -> dict:
|
||||
"requested_at": iso(inv.requested_at),
|
||||
"decided_at": iso(inv.decided_at),
|
||||
"decided_by": inv.decided_by,
|
||||
# When (if ever) the admin sent this person an invitation. Adding an address is silent, so
|
||||
# without this the admin has no way to answer "did I already write to them?" — and the only
|
||||
# failure mode is writing to a real person twice.
|
||||
"invited_at": iso(inv.invited_at),
|
||||
# Drives the delete affordance: an entry backed by an account is not removable here.
|
||||
"has_account": has_account,
|
||||
# …and whether removing it would actually take access away, or leave the env allow-list
|
||||
# granting it anyway (for an ADMIN_EMAILS address, as an admin).
|
||||
"still_allowed_by_env": _env_allowed(inv.email),
|
||||
# The language this person asked us in, or null when nobody ever told us (a hand-added
|
||||
# whitelist row). The invitation dialog preselects it, and falls back to the admin's own UI
|
||||
# language — a guess, but a visible one the admin can correct before sending.
|
||||
"lang": inv.lang,
|
||||
}
|
||||
|
||||
|
||||
@@ -56,32 +110,77 @@ def list_invites(
|
||||
rows = db.execute(stmt.order_by(Invite.requested_at.desc())).scalars().all()
|
||||
order = {"pending": 0, "approved": 1, "denied": 2}
|
||||
rows.sort(key=lambda i: order.get(i.status, 9))
|
||||
return [_serialize(i) for i in rows]
|
||||
accounts = _accounts_for(db, (i.email for i in rows))
|
||||
return [_serialize(i, has_account=canonical_email(i.email) in accounts) for i in rows]
|
||||
|
||||
|
||||
def _welcome_mail_parts(db: Session, email: str) -> tuple[str | None, bool, int]:
|
||||
"""What a welcome or invitation mail should tell this address to do next: a confirmation link (if
|
||||
there is an unconfirmed account behind it), whether an account exists at all, and how long the
|
||||
link lasts.
|
||||
|
||||
This is the C option's whole mechanic. In the common ordering — the address is approved and the
|
||||
person signs up either side of that — they would otherwise have to find and open TWO messages:
|
||||
one saying they are in, another asking them to confirm. Carrying the link inside the mail that is
|
||||
already going out makes it one mail and one click.
|
||||
|
||||
Resolved through `find_user_by_email`, so an account registered under another spelling of the
|
||||
same Gmail mailbox is still recognised — and, deliberately, so several candidates resolve to
|
||||
"no single answer" rather than a guess: an invitation that carried the wrong account's link would
|
||||
be worse than one that simply pointed at the sign-up page.
|
||||
"""
|
||||
hours = int(APPROVAL_VERIFY_TTL.total_seconds() // 3600)
|
||||
user = find_user_by_email(db, email)
|
||||
if user is None or user.is_demo:
|
||||
return None, False, hours
|
||||
if user.email_verified or not email_mod.email_enabled(db):
|
||||
return None, True, hours
|
||||
return issue_verification_link(db, user, APPROVAL_VERIFY_TTL), True, hours
|
||||
|
||||
|
||||
def _activate_user_by_email(db: Session, email: str) -> None:
|
||||
"""Approving access activates a matching (e.g. pending password-registration) account so
|
||||
it can sign in. No-op if no such user exists yet (a Google user is created at first login)."""
|
||||
user = db.execute(select(User).where(User.email == email.lower())).scalar_one_or_none()
|
||||
it can sign in. No-op if no such user exists yet (a Google user is created at first login).
|
||||
|
||||
E9.5 S1: this path deliberately does NOT revoke a preset password. Revoking here destroyed the
|
||||
password a legitimate registrant had just chosen and made "your password was cleared for
|
||||
security" the DEFAULT experience, because approve-before-verify is the normal ordering.
|
||||
|
||||
S1 justified that by saying `password_login` refuses an unverified account BEFORE it looks at
|
||||
`is_active`, so a pre-emptor could not sign in "approved or not". A review of the finished epic
|
||||
showed that argument only holds while the address stays unverified — and S6's approval mail
|
||||
hands the real owner a confirmation link precisely so they will end that state. Nothing replaced
|
||||
the revoke, and that is deliberate: round 2 of the same review tried "the newest registration
|
||||
owns an unverified account" and reverted it, because the rule is symmetric — an attacker can
|
||||
always register last — and it handed anonymous callers a write onto a real account's credential.
|
||||
The owner's recourse is password-reset, which proves the mailbox, rewrites the hash and bumps
|
||||
the session epoch. The residual is logged as SEC-16. The revoke also stays on the Google-adoption path
|
||||
(auth.revoke_preset_password_if_unverified), where ownership is actually attested."""
|
||||
# Canonical, with the literal spelling preferred when both exist: approving
|
||||
# `npeter83.temu@gmail.com` must activate the account that registered as `npeter83temu@gmail.com`,
|
||||
# since Gmail delivers both to one mailbox and the admin decided about the mailbox.
|
||||
email = email.lower()
|
||||
candidates = (
|
||||
db.execute(
|
||||
select(User).where(User.email_canonical == canonical_email(email)).order_by(User.id)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
user = next((u for u in candidates if u.email == email), None) or next(iter(candidates), None)
|
||||
if user is not None and not user.is_active:
|
||||
# SEC-10: same guard as the Google-adoption path — approving an account whose email was never
|
||||
# verified must not hand a working login to a password set on it by whoever registered the
|
||||
# address (a pre-emption). Shared single-source helper so the two paths can't drift.
|
||||
revoke_preset_password_if_unverified(user)
|
||||
user.is_active = True
|
||||
db.commit()
|
||||
db.flush() # the handler commits (CLAUDE.md: helpers flush)
|
||||
|
||||
|
||||
def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
inv = db.get(Invite, invite_id)
|
||||
if inv is None:
|
||||
raise HTTPException(status_code=404, detail="No such invite")
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
# Idempotency guard: only a still-pending invite can be decided. Without this a repeat/stale
|
||||
# approve or deny re-runs its side effects — re-activating a user, or (now that deny emails)
|
||||
# sending a duplicate decision mail, even flipping an already-approved invite to "denied" and
|
||||
# mailing a rejection to an already-active account.
|
||||
if inv.status != "pending":
|
||||
raise HTTPException(status_code=409, detail="Invite already decided")
|
||||
raise refused(409, "invite_decided")
|
||||
inv.status = "approved" if approved else "denied"
|
||||
inv.decided_at = datetime.now(timezone.utc)
|
||||
inv.decided_by = admin.email
|
||||
@@ -93,7 +192,7 @@ def _decide(db: Session, invite_id: int, admin: User, approved: bool) -> Invite:
|
||||
target_id=inv.id,
|
||||
summary=inv.email,
|
||||
)
|
||||
db.commit()
|
||||
db.flush() # the handler commits, with the activation that goes with it
|
||||
return inv
|
||||
|
||||
|
||||
@@ -106,8 +205,16 @@ def approve_invite(
|
||||
) -> dict:
|
||||
inv = _decide(db, invite_id, admin, approved=True)
|
||||
_activate_user_by_email(db, inv.email)
|
||||
background.add_task(email_mod.send_access_approved, inv.email)
|
||||
return _serialize(inv)
|
||||
# Computed here, not in the background task: issuing the token needs this request's session, and
|
||||
# the task must not outlive it. Approval answers a request the person MADE, so unlike the
|
||||
# invitation below it is sent automatically — a reply is owed, not a nudge.
|
||||
link, has_account, hours = _welcome_mail_parts(db, inv.email)
|
||||
lang = mailtext.lang_for_email(db, inv.email)
|
||||
background.add_task(email_mod.send_access_approved, inv.email, lang, link, has_account, hours)
|
||||
# Before the response, so the mail (a background task, after it) never describes a decision
|
||||
# the database does not hold yet.
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [inv.email])))
|
||||
|
||||
|
||||
@router.post("/invites/{invite_id}/deny")
|
||||
@@ -120,8 +227,11 @@ def deny_invite(
|
||||
# Symmetry with approve: the requester hears back either way, instead of a silent limbo. No
|
||||
# account is activated, so this is an email only (the requester has no in-app inbox yet).
|
||||
inv = _decide(db, invite_id, admin, approved=False)
|
||||
background.add_task(email_mod.send_access_denied, inv.email)
|
||||
return _serialize(inv)
|
||||
background.add_task(
|
||||
email_mod.send_access_denied, inv.email, mailtext.lang_for_email(db, inv.email)
|
||||
)
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [inv.email])))
|
||||
|
||||
|
||||
class AddInviteIn(BaseModel):
|
||||
@@ -137,20 +247,145 @@ def add_invite(
|
||||
"""Manually whitelist an email (approved straight away). Convenience for the admin."""
|
||||
email = body.email.strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
inv = db.execute(select(Invite).where(Invite.email == email)).scalar_one_or_none()
|
||||
raise refused(400, "invalid_email")
|
||||
# Reuse a canonically-equal row instead of adding a second one for the same mailbox. Prod carries
|
||||
# exactly that pair for one person from 2026-06-16 (hartmann.peter1986@ / hartmannpeter1986@),
|
||||
# which is how an admin ends up approving the same human twice and still seeing them bounce.
|
||||
canon = canonical_email(email)
|
||||
rows = (
|
||||
db.execute(select(Invite).where(Invite.email_canonical == canon).order_by(Invite.id))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
inv = next((r for r in rows if r.email == email), None) or next(iter(rows), None)
|
||||
if inv is None:
|
||||
inv = Invite(email=email)
|
||||
db.add(inv)
|
||||
inv.status = "approved"
|
||||
inv.decided_at = datetime.now(timezone.utc)
|
||||
inv.decided_by = admin.email
|
||||
db.flush() # a new invite's id, for the audit row — before it, the row named no target
|
||||
audit.record(
|
||||
db, admin.id, AuditAction.INVITE_ADD, target_type="invite", target_id=inv.id, summary=email
|
||||
)
|
||||
db.commit()
|
||||
# One transaction: the whitelisting and the activation it implies land together or not at all.
|
||||
_activate_user_by_email(db, email)
|
||||
return _serialize(inv)
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=bool(_accounts_for(db, [email])))
|
||||
|
||||
|
||||
class InviteSendIn(BaseModel):
|
||||
# Which language to write this person in. Optional so an existing caller (and a test) can post
|
||||
# an empty body, but the admin UI always sends it: this is the one recipient the app has no
|
||||
# signal about, because the admin typed the address in themselves.
|
||||
lang: str | None = None
|
||||
|
||||
|
||||
@router.post("/invites/{invite_id}/invite")
|
||||
def send_invitation(
|
||||
invite_id: int,
|
||||
body: InviteSendIn | None = None,
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Tell someone their address is whitelisted — only when the admin asks for it.
|
||||
|
||||
Whitelisting stays SILENT on purpose. The operator opens a door as a possibility, usually having
|
||||
already agreed it with the person, and an address can sit approved for weeks with nobody
|
||||
registering; a mail fired automatically would be nudging someone who never asked to be nudged.
|
||||
So the send is its own act, on its own button, and `invited_at` records it so the admin can see
|
||||
they have already written rather than writing twice.
|
||||
|
||||
Only an APPROVED row can be invited: there is nothing to invite someone to while their request is
|
||||
undecided, and inviting a denied address would contradict the decision."""
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
if inv.status != "approved":
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "invite_send", "reason": "not_approved", "email": inv.email},
|
||||
)
|
||||
if not email_mod.email_enabled(db):
|
||||
# Same reasoning as the admin-side verification resend: mail is fail-soft everywhere else,
|
||||
# but a button the admin pressed must not answer with a success toast for a mail that was
|
||||
# never sent.
|
||||
raise HTTPException(status_code=409, detail={"code": "invite_send", "reason": "email_off"})
|
||||
link, has_account, hours = _welcome_mail_parts(db, inv.email)
|
||||
# An explicit pick from the dialog is recorded, so a later mail to this address matches the one
|
||||
# the admin already sent. Only the explicit pick — writing the fallback back would turn a column
|
||||
# that means "what this person told us" into "what we last guessed".
|
||||
picked = body.lang if body else None
|
||||
if picked in mailtext.SUPPORTED:
|
||||
lang = inv.lang = picked
|
||||
else:
|
||||
lang = mailtext.lang_for_email(db, inv.email)
|
||||
# Sent INLINE, so `invited_at` records the send rather than the intent. Every other mail here is
|
||||
# fail-soft in the background, which is right when nobody is waiting on it — but this column
|
||||
# exists to answer "did I already write to them?", and the UI dims the button and warns about a
|
||||
# second mail on the strength of it. Stamping it before a send that then failed makes the one
|
||||
# question it answers a lie, and talks the admin out of the retry that would fix it.
|
||||
#
|
||||
# One thing a failed send does leave behind: the confirmation token `_welcome_mail_parts` minted
|
||||
# to render the link, which commits inside `_issue_token`. Accepted rather than chased — the
|
||||
# token is single-use, expires on its own, and was mailed nowhere; deferring it would mean
|
||||
# rewriting a helper three callers share so the handler owns its commit.
|
||||
if not email_mod.send_access_invitation(inv.email, lang, link, has_account, hours):
|
||||
raise HTTPException(
|
||||
status_code=422, detail={"code": "invite_send", "reason": "send_failed"}
|
||||
)
|
||||
# Only after the mail is actually out. Same as the other two paths that decide for an address
|
||||
# (`approve_invite`, `add_invite`): an approved row means this mailbox may come in, so finish it,
|
||||
# or the mail can promise "your account is ready — just sign in" to someone `password_login` will
|
||||
# refuse for `is_active`. Doing it BEFORE the send would mean a request the UI reports as failed
|
||||
# had quietly activated an account behind the admin's back.
|
||||
_activate_user_by_email(db, inv.email)
|
||||
inv.invited_at = datetime.now(timezone.utc)
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.INVITE_SEND,
|
||||
target_type="invite",
|
||||
target_id=inv.id,
|
||||
summary=inv.email,
|
||||
)
|
||||
db.commit()
|
||||
return _serialize(inv, has_account=has_account)
|
||||
|
||||
|
||||
@router.delete("/invites/{invite_id}")
|
||||
def delete_invite(
|
||||
invite_id: int,
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Remove a whitelist/request entry outright — the counterpart to add_invite, for an address that
|
||||
never became an account (an approval added by mistake, or one that is no longer wanted). Deny
|
||||
records a decision and keeps the row for the audit trail; delete forgets the address. Neither
|
||||
stops the person asking again: `upsert_pending_invite` reopens a denied row, and a deleted one
|
||||
simply comes back as a fresh pending request.
|
||||
|
||||
Refused once an account exists for the address, whatever its state: this row is the account's
|
||||
approval gate, and orphaning the account from it would leave a person who can neither sign in nor
|
||||
be approved. Delete the USER instead — purge_user removes the invite along with it.
|
||||
|
||||
Reports `still_allowed_by_env`, because deleting the row is NOT always a revoke: `is_allowed`
|
||||
falls back to the env ALLOWED_EMAILS / ADMIN_EMAILS bootstrap lists once no invite exists, so for
|
||||
an address on those the deletion leaves access intact (and an ADMIN_EMAILS address is still
|
||||
promoted to admin on its next Google sign-in). The admin has to hear that, not guess it."""
|
||||
inv = get_or_404(db, Invite, invite_id, what="invite")
|
||||
if _accounts_for(db, [inv.email]):
|
||||
raise refused(409, "address_has_account")
|
||||
still_allowed_by_env = _env_allowed(inv.email)
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.INVITE_DELETE,
|
||||
target_type="invite",
|
||||
target_id=inv.id,
|
||||
summary=inv.email,
|
||||
)
|
||||
db.delete(inv)
|
||||
db.commit()
|
||||
return {"deleted": invite_id, "still_allowed_by_env": still_allowed_by_env}
|
||||
|
||||
|
||||
# --- Users & roles --------------------------------------------------------------------
|
||||
@@ -167,8 +402,18 @@ def _serialize_user(u: User) -> dict:
|
||||
"email_verified": u.email_verified,
|
||||
"is_demo": u.is_demo,
|
||||
"has_password": u.password_hash is not None,
|
||||
# Distinguishes "this account never had a password" (Google-only) from "SEC-10 cleared it" —
|
||||
# the same distinction _send_reset_if_eligible needs, and the one an admin has to see before
|
||||
# deciding what kind of help someone needs.
|
||||
"password_revoked_at": iso(u.password_revoked_at),
|
||||
"has_google": u.google_sub is not None,
|
||||
# Only when it DIFFERS from the account address (E9.5 S3): "connected" is not the same
|
||||
# statement as "connected as someone else", and the second one is the one that explains a
|
||||
# confusing sign-in.
|
||||
"google_email": u.google_email if u.google_email != u.email else None,
|
||||
"created_at": iso(u.created_at),
|
||||
# NULL = no sign-in since the column existed (0085), or the shared demo account.
|
||||
"last_login_at": iso(u.last_login_at),
|
||||
}
|
||||
|
||||
|
||||
@@ -194,21 +439,19 @@ def set_user_role(
|
||||
account, and can't remove the last remaining admin. Emails the user when it actually changes."""
|
||||
role = body.role
|
||||
if role not in ("user", "admin"):
|
||||
raise HTTPException(status_code=400, detail="role must be 'user' or 'admin'")
|
||||
target = db.get(User, user_id)
|
||||
if target is None:
|
||||
raise HTTPException(status_code=404, detail="No such user")
|
||||
raise refused(400, "invalid_role")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account's role can't be changed.")
|
||||
raise refused(400, "demo_role")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="You can't change your own role.")
|
||||
raise refused(400, "own_role")
|
||||
if (
|
||||
target.role == "admin"
|
||||
and role == "user"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't remove the last admin.")
|
||||
raise refused(400, "last_admin")
|
||||
changed = target.role != role
|
||||
if changed:
|
||||
audit.record(
|
||||
@@ -224,7 +467,13 @@ def set_user_role(
|
||||
target.role = role
|
||||
db.commit()
|
||||
if changed:
|
||||
background.add_task(email_mod.send_role_changed, target.email, role, operator_contact())
|
||||
background.add_task(
|
||||
email_mod.send_role_changed,
|
||||
target.email,
|
||||
mailtext.lang_for_user(target),
|
||||
role,
|
||||
operator_contact(),
|
||||
)
|
||||
return _serialize_user(target)
|
||||
|
||||
|
||||
@@ -246,20 +495,18 @@ def set_user_suspended(
|
||||
un-suspend the user is emailed they can return (suspend is announced reactively, on a blocked
|
||||
sign-in)."""
|
||||
suspended = body.suspended
|
||||
target = db.get(User, user_id)
|
||||
if target is None:
|
||||
raise HTTPException(status_code=404, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be suspended.")
|
||||
raise refused(400, "demo_suspend")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(status_code=400, detail="You can't suspend your own account.")
|
||||
raise refused(400, "own_suspend")
|
||||
if (
|
||||
suspended
|
||||
and target.role == "admin"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't suspend the last active admin.")
|
||||
raise refused(400, "last_active_admin")
|
||||
was_suspended = target.is_suspended
|
||||
if was_suspended != suspended:
|
||||
audit.record(
|
||||
@@ -273,7 +520,12 @@ def set_user_suspended(
|
||||
target.is_suspended = suspended
|
||||
db.commit()
|
||||
if was_suspended and not suspended:
|
||||
background.add_task(email_mod.send_account_unsuspended, target.email, operator_contact())
|
||||
background.add_task(
|
||||
email_mod.send_account_unsuspended,
|
||||
target.email,
|
||||
mailtext.lang_for_user(target),
|
||||
operator_contact(),
|
||||
)
|
||||
return _serialize_user(target)
|
||||
|
||||
|
||||
@@ -287,21 +539,17 @@ def admin_delete_user(
|
||||
"""Admin-triggered account deletion — the same full erasure a user can perform on themselves
|
||||
(cascade delete of all personal data + access-request cleanup + Google-grant revoke). Guards:
|
||||
not the demo account, not yourself (use Settings → Account), and not the last admin."""
|
||||
target = db.get(User, user_id)
|
||||
if target is None:
|
||||
raise HTTPException(status_code=404, detail="No such user")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(status_code=400, detail="The demo account can't be deleted.")
|
||||
raise refused(400, "demo_delete")
|
||||
if target.id == admin.id:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="Delete your own account from Settings → Account."
|
||||
)
|
||||
raise refused(400, "own_delete")
|
||||
if (
|
||||
target.role == "admin"
|
||||
and not target.is_suspended
|
||||
and count_admins(db, active_only=True) <= 1
|
||||
):
|
||||
raise HTTPException(status_code=400, detail="Can't delete the last admin.")
|
||||
raise refused(400, "last_admin")
|
||||
# Record BEFORE the purge — the target row (and its email) is gone afterwards.
|
||||
audit.record(
|
||||
db,
|
||||
@@ -318,6 +566,197 @@ def admin_delete_user(
|
||||
# --- Demo account: email whitelist + state reset -------------------------------------
|
||||
|
||||
|
||||
# --- Stuck sign-ups: the admin lifeline (E9.5 S5, roadmap UX-25) ---------------------------------
|
||||
|
||||
|
||||
def _pending_access_rows(db: Session) -> list[User]:
|
||||
"""Accounts that have signed up but still cannot sign in, newest first.
|
||||
|
||||
Two different dead ends share this list, because from the admin's chair they look the same — a
|
||||
person who registered and is still outside:
|
||||
· `is_active=False` — approved BEFORE they signed up, so nothing ever landed in the
|
||||
Access-requests queue and its Approve button doesn't apply (`_decide` only takes a pending
|
||||
row). This is roadmap UX-25, and it is why the list exists.
|
||||
· `email_verified=False` — the confirmation mail hasn't been acted on. Not the admin's to
|
||||
decide, but theirs to help with: on 2026-09-03 the first mail to a fresh Gmail landed in
|
||||
Spam, so "send it again" is a real move.
|
||||
A suspended account is deliberately excluded: it cannot sign in either, but that is a decision
|
||||
already made, not a person stuck waiting on one.
|
||||
"""
|
||||
return list(
|
||||
db.execute(
|
||||
select(User)
|
||||
.where(
|
||||
User.is_demo.is_(False),
|
||||
User.is_suspended.is_(False),
|
||||
or_(User.is_active.is_(False), User.email_verified.is_(False)),
|
||||
)
|
||||
.order_by(User.created_at.desc())
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/pending-access")
|
||||
def list_pending_access(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
"""The sign-ups that are still outside, with which gate is shut for each. `can_activate` cannot
|
||||
be derived on the client — it needs the invite/allow-list lookup — so it is answered here.
|
||||
|
||||
One `is_allowed` query per row rather than a batched re-implementation of it: the row count is
|
||||
the admin's own backlog of stuck sign-ups (single digits on any real instance), and a second,
|
||||
subtly different copy of the "may this address sign in" rule is exactly the drift this codebase
|
||||
keeps getting bitten by."""
|
||||
out = []
|
||||
for u in _pending_access_rows(db):
|
||||
# The whole rule in one line: this address MAY sign in, and this account still can't. An
|
||||
# undecided request fails it (nothing is allowed yet) and so does a denied one — both belong
|
||||
# to the queue above, which owns that decision. Without this, every fresh registration
|
||||
# appeared in both lists at once, which UAT caught on 2026-09-05.
|
||||
if not is_allowed(db, u.email):
|
||||
continue
|
||||
out.append(_serialize_user(u) | {"can_activate": can_activate(db, u)})
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/users/{user_id}/access")
|
||||
def user_access(
|
||||
user_id: int,
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Everything the account panel shows about why this person can or cannot get in (E12 S5).
|
||||
|
||||
Its own endpoint rather than extra columns on /users: `is_allowed` and `can_activate` each run a
|
||||
query, and the panel opens for ONE user at a time — folding them into the list would pay that cost
|
||||
per row on every load to answer a question nobody asked yet."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
return _serialize_user(target) | {
|
||||
"is_allowed": is_allowed(db, target.email),
|
||||
"can_activate": can_activate(db, target),
|
||||
"email_configured": email_mod.email_enabled(db),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/users/{user_id}/set-password-link")
|
||||
def create_set_password_link(
|
||||
user_id: int,
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Mint a one-time link that lets this account set a password, with no email involved (E12 S5).
|
||||
|
||||
The answer to SEC-15, SEC-16 and "no SMTP and someone forgot their password" at once. The admin
|
||||
hands the link over by whatever channel they trust; opening it sets a password and signs out
|
||||
whoever held the old one.
|
||||
|
||||
Deliberately NOT gated on the account's state. It is useful for a pre-empted address, for a
|
||||
revoked password, for a Google-only account that wants one, and for a plain forgotten password on
|
||||
an instance with no mail — narrowing it to one of those would only make the admin's real case the
|
||||
one it refuses. The demo account is the single exception: it is shared, so handing anyone a way to
|
||||
change its password is handing them everyone else's session."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "setpw", "reason": "demo", "email": target.email}
|
||||
)
|
||||
url, expires_at = issue_set_password_link(db, target)
|
||||
# The URL itself is NOT recorded: the audit log is read by humans and kept long, and a live
|
||||
# credential-setting link is the last thing that belongs in it. That the link was issued, by whom,
|
||||
# for whom, is the auditable fact.
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.USER_SETPW_LINK,
|
||||
target_type="user",
|
||||
target_id=target.id,
|
||||
summary=target.email,
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "url": url, "expires_at": iso(expires_at), "email": target.email}
|
||||
|
||||
|
||||
@router.post("/users/{user_id}/activate")
|
||||
def activate_user(
|
||||
user_id: int,
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Let in a registrant whose address was approved before they signed up (roadmap UX-25).
|
||||
|
||||
Admin-side ONLY, and that is the whole design. Two earlier attempts at this were reverted after
|
||||
review, and both moved authority rather than visibility: auto-activating on registration turns a
|
||||
standing whitelist entry into a bearer grant for whoever registers the address first, and
|
||||
reopening the approved invite to `pending` lets an anonymous sign-up REVOKE a standing approval.
|
||||
This endpoint changes nothing about what an unauthenticated caller can cause — the admin could
|
||||
already do exactly this by re-entering the address in the whitelist box (`add_invite` →
|
||||
`_activate_user_by_email`). What was missing was any way to know they had to."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise refused(400, "demo_activate")
|
||||
if target.is_active:
|
||||
return _serialize_user(target) # idempotent: a double click is not an error
|
||||
if not is_allowed(db, target.email):
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "activate", "reason": "not_allowed", "email": target.email},
|
||||
)
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.USER_ACTIVATE,
|
||||
target_type="user",
|
||||
target_id=target.id,
|
||||
summary=target.email,
|
||||
before={"is_active": False},
|
||||
after={"is_active": True},
|
||||
)
|
||||
target.is_active = True
|
||||
db.commit()
|
||||
return _serialize_user(target)
|
||||
|
||||
|
||||
@router.post("/users/{user_id}/resend-verification")
|
||||
def resend_user_verification(
|
||||
user_id: int,
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Re-send an account's confirmation link from the admin side.
|
||||
|
||||
The public /auth/verify/resend answers uniformly and reveals nothing, which is right for an
|
||||
anonymous caller but leaves the admin unable to tell a delivered mail from a dropped one. Here
|
||||
the caller is already authenticated as an admin, so this one says plainly what happened.
|
||||
|
||||
Not rate-limited: an admin is not a threat model to enumerate against, and the deliberate silence
|
||||
of the public endpoint is exactly what this exists to escape."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo or target.email_verified:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "resend", "reason": "not_applicable", "email": target.email},
|
||||
)
|
||||
if not email_mod.email_enabled(db):
|
||||
# Mail is fail-soft everywhere else, which is what left the 0.78.5 sign-ups with nothing to
|
||||
# fall back on. An admin pressing a button deserves the failure, not a success toast.
|
||||
raise HTTPException(status_code=409, detail={"code": "resend", "reason": "email_off"})
|
||||
if not send_verification_link(db, target):
|
||||
# Same rule as the invitation above: a button an admin pressed must not answer with a
|
||||
# success for a mail that was dropped. Mail is fail-soft everywhere else because nobody is
|
||||
# waiting on the answer; here somebody is.
|
||||
raise HTTPException(status_code=422, detail={"code": "resend", "reason": "send_failed"})
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.USER_VERIFY_RESEND,
|
||||
target_type="user",
|
||||
target_id=target.id,
|
||||
summary=target.email,
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "email": target.email}
|
||||
|
||||
|
||||
def _serialize_demo(row: DemoWhitelist) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
@@ -351,7 +790,7 @@ def add_demo_whitelist(
|
||||
sign-in). Idempotent — re-adding an existing email just returns it."""
|
||||
email = body.email.strip().lower()
|
||||
if not valid_email(email):
|
||||
raise HTTPException(status_code=400, detail="Enter a valid email address.")
|
||||
raise refused(400, "invalid_email")
|
||||
row = db.execute(select(DemoWhitelist).where(DemoWhitelist.email == email)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = DemoWhitelist(
|
||||
@@ -398,7 +837,7 @@ def reset_demo(admin: User = Depends(admin_user), db: Session = Depends(get_db))
|
||||
db, admin.id, AuditAction.DEMO_RESET, summary=f"{seeded} sample playlists re-seeded"
|
||||
)
|
||||
db.commit()
|
||||
return {"reset": True, "playlists_seeded": seeded}
|
||||
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
||||
|
||||
|
||||
@router.post("/purge-discovery")
|
||||
@@ -417,4 +856,4 @@ def purge_discovery(admin: User = Depends(admin_user), db: Session = Depends(get
|
||||
after=result,
|
||||
)
|
||||
db.commit()
|
||||
return result
|
||||
return {"ok": True, **result}
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""The admin diagnostics console's read side (E22.6): query `log_events`, the server's stored log.
|
||||
|
||||
Admin-only and read-only — rows leave the table through the retention job (`log_retention_days`),
|
||||
never through here. Reading is not audited: it changes nothing, and the console's live tail polls,
|
||||
so a row per read would bury the audit log's real entries.
|
||||
|
||||
What a row holds was redacted when it was written (`logs.record_fields`, and the ingest route for a
|
||||
browser's rows), so this serves the columns as they are; a value the redactor took out reads as
|
||||
`[redacted]` / `[email]`, which is the point — the console shows that something was there.
|
||||
|
||||
Paging is a keyset over `(ts desc, id desc)`: a browser row's `ts` is clamped into its consent
|
||||
window, so `ts` alone is not unique, and an offset would skip or repeat rows while the writer adds
|
||||
new ones at the top. The live tail re-reads the first page rather than asking for "ids after N":
|
||||
several processes write this table, and a batch's ids are taken when it inserts, not when it
|
||||
commits, so a slower batch can land below an id the tail already passed — and would never show.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import Text, and_, cast, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logsink
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import LogEvent, User
|
||||
from app.utils import like_escape
|
||||
|
||||
router = APIRouter(prefix="/api/admin/logs", tags=["admin"])
|
||||
|
||||
PAGE_MAX = 200
|
||||
# A facet list is a chip row, not a report: past this many distinct events the rest are reachable
|
||||
# through the free-text search, and the response stays small whatever a self-host logs.
|
||||
FACET_EVENTS_MAX = 100
|
||||
|
||||
Level = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
# The filters a facet can leave out of its own count (`LogFilters.conditions`).
|
||||
Facet = Literal["levels", "sources", "user", "event"]
|
||||
# `none` selects the rows that have no account / no event code (a scheduler line, an anonymous
|
||||
# request) — a real value to filter on, not the absence of a filter.
|
||||
NONE = "none"
|
||||
|
||||
|
||||
def _utc(value: datetime | None) -> datetime | None:
|
||||
"""A bound without a zone is UTC — the column is `timestamptz`, and a naive value would be read
|
||||
in the database session's zone instead."""
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _encode_cursor(row: LogEvent) -> str:
|
||||
return base64.urlsafe_b64encode(json.dumps([row.ts.isoformat(), row.id]).encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(cursor: str) -> tuple[datetime, int]:
|
||||
try:
|
||||
ts, row_id = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
return _utc(datetime.fromisoformat(ts)), int(row_id)
|
||||
except (binascii.Error, json.JSONDecodeError, ValueError, TypeError) as exc:
|
||||
raise HTTPException(status_code=400, detail={"code": "bad_cursor"}) from exc
|
||||
|
||||
|
||||
def _accounts(db: Session, user_ids: set[int]) -> dict[str, str]:
|
||||
"""The accounts a page names, by id → address. An id with no account left (erased after the
|
||||
line was written) is simply absent, and the client shows the bare id."""
|
||||
if not user_ids:
|
||||
return {}
|
||||
rows = db.execute(select(User.id, User.email).where(User.id.in_(user_ids))).all()
|
||||
return {str(uid): email for uid, email in rows}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LogFilters:
|
||||
"""The console's filters, shared by the page and its facet counts."""
|
||||
|
||||
levels: list[str]
|
||||
sources: list[str]
|
||||
user: str | None
|
||||
event: str | None
|
||||
request_id: str | None
|
||||
job_id: int | None
|
||||
q: str | None
|
||||
since: datetime | None
|
||||
until: datetime | None
|
||||
|
||||
def conditions(self, skip: Facet | None = None) -> list:
|
||||
"""The WHERE conditions, ANDed by the caller. `skip` leaves one filter out: a facet counts
|
||||
its own values under every OTHER filter, so picking a level does not zero the other
|
||||
levels' counts (the way back), while a request pivot does narrow them (honest counts)."""
|
||||
out = []
|
||||
if self.levels and skip != "levels":
|
||||
out.append(LogEvent.level.in_(self.levels))
|
||||
if self.sources and skip != "sources":
|
||||
out.append(LogEvent.source.in_(self.sources))
|
||||
if self.user is not None and skip != "user":
|
||||
out.append(
|
||||
LogEvent.user_id.is_(None)
|
||||
if self.user == NONE
|
||||
else LogEvent.user_id == int(self.user)
|
||||
)
|
||||
if self.event is not None and skip != "event":
|
||||
out.append(
|
||||
LogEvent.event.is_(None) if self.event == NONE else LogEvent.event == self.event
|
||||
)
|
||||
if self.request_id is not None:
|
||||
out.append(LogEvent.request_id == self.request_id)
|
||||
if self.job_id is not None:
|
||||
out.append(LogEvent.job_id == self.job_id)
|
||||
if self.since is not None:
|
||||
out.append(LogEvent.ts >= _utc(self.since))
|
||||
if self.until is not None:
|
||||
out.append(LogEvent.ts < _utc(self.until))
|
||||
if self.q and self.q.strip():
|
||||
like = f"%{like_escape(self.q.strip())}%"
|
||||
out.append(
|
||||
or_(
|
||||
LogEvent.msg.ilike(like),
|
||||
LogEvent.event.ilike(like),
|
||||
LogEvent.request_id.ilike(like),
|
||||
LogEvent.exc.ilike(like),
|
||||
cast(LogEvent.attrs, Text).ilike(like),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def log_filters(
|
||||
levels: list[Level] = Query(default=[]),
|
||||
sources: list[str] = Query(default=[], max_length=8),
|
||||
user: str | None = Query(default=None, pattern=r"^(none|[0-9]{1,10})$"),
|
||||
event: str | None = Query(
|
||||
default=None, max_length=64, pattern=r"^(none|[a-z][a-z0-9_]*(\.[a-z0-9_]+)*)$"
|
||||
),
|
||||
request_id: str | None = Query(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$"),
|
||||
job_id: int | None = Query(default=None, ge=1),
|
||||
q: str | None = Query(default=None, max_length=200),
|
||||
since: datetime | None = None,
|
||||
until: datetime | None = None,
|
||||
) -> LogFilters:
|
||||
return LogFilters(list(levels), list(sources), user, event, request_id, job_id, q, since, until)
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_logs(
|
||||
filters: LogFilters = Depends(log_filters),
|
||||
before: str | None = Query(default=None, max_length=200),
|
||||
limit: int = Query(default=100, ge=1, le=PAGE_MAX),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""One page of rows, newest first, matching every filter given (they AND together); `before`
|
||||
continues from a previous page's `next_cursor`."""
|
||||
query = select(LogEvent).where(*filters.conditions())
|
||||
if before is not None:
|
||||
ts, row_id = _decode_cursor(before)
|
||||
# The `ts <=` conjunct is implied by the OR; it is there so the planner can bound the
|
||||
# `ts` index scan instead of filtering every newer row first.
|
||||
query = query.where(
|
||||
and_(
|
||||
LogEvent.ts <= ts,
|
||||
or_(LogEvent.ts < ts, and_(LogEvent.ts == ts, LogEvent.id < row_id)),
|
||||
)
|
||||
)
|
||||
|
||||
# One more than asked, to know whether there is a next page without a second count query.
|
||||
rows = (
|
||||
db.execute(query.order_by(LogEvent.ts.desc(), LogEvent.id.desc()).limit(limit + 1))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return {
|
||||
"items": [logsink.serialize(r) for r in rows],
|
||||
"accounts": _accounts(db, {r.user_id for r in rows if r.user_id is not None}),
|
||||
"next_cursor": _encode_cursor(rows[-1]) if more else None,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/facets")
|
||||
def log_facets(
|
||||
filters: LogFilters = Depends(log_filters),
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""What the filter chips offer, with counts: the levels, sources and accounts present, and the
|
||||
most frequent events — each counted under every filter but its own (`LogFilters.conditions`).
|
||||
`total` is the "all levels" count: every filter but the level one."""
|
||||
|
||||
def counts(col, skip: Facet, cap: int | None = None) -> list[dict]:
|
||||
n = func.count().label("n")
|
||||
stmt = select(col, n).where(*filters.conditions(skip)).group_by(col).order_by(n.desc(), col)
|
||||
if cap is not None:
|
||||
stmt = stmt.limit(cap)
|
||||
return [
|
||||
{"value": NONE if value is None else value, "count": count}
|
||||
for value, count in db.execute(stmt).all()
|
||||
]
|
||||
|
||||
users = counts(LogEvent.user_id, "user")
|
||||
total = select(func.count()).select_from(LogEvent).where(*filters.conditions("levels"))
|
||||
return {
|
||||
"levels": counts(LogEvent.level, "levels"),
|
||||
"sources": counts(LogEvent.source, "sources"),
|
||||
"events": counts(LogEvent.event, "event", FACET_EVENTS_MAX),
|
||||
"users": [{**u, "value": str(u["value"])} for u in users],
|
||||
"accounts": _accounts(db, {u["value"] for u in users if u["value"] != NONE}),
|
||||
"total": db.scalar(total) or 0,
|
||||
}
|
||||
@@ -68,4 +68,4 @@ def clear_audit(admin: User = Depends(admin_user), db: Session = Depends(get_db)
|
||||
n = db.execute(delete(AuditLog)).rowcount or 0
|
||||
audit.record(db, admin.id, AuditAction.AUDIT_CLEAR, summary=f"Cleared {n} audit entries")
|
||||
db.commit()
|
||||
return {"cleared": n}
|
||||
return {"ok": True, "cleared": n}
|
||||
|
||||
+584
-190
File diff suppressed because it is too large
Load Diff
@@ -2,15 +2,14 @@
|
||||
hidden, tags) and per-channel sync state. Local reads/writes plus YouTube-side writes
|
||||
(subscribe / unsubscribe) that require the optional write scope."""
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import quota
|
||||
from app import logs, quota
|
||||
from app.auth import admin_user, current_user, require_human, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
@@ -26,16 +25,17 @@ from app.models import (
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.routes._common import owned_or_404
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404, youtube_refusal
|
||||
from app.sync.explore import explore_ingest_page
|
||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/channels", tags=["channels"])
|
||||
|
||||
log = logging.getLogger("siftlode.api")
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
@router.get("")
|
||||
@@ -166,7 +166,15 @@ def discover_channels(
|
||||
user.yt_channel_id = own
|
||||
db.commit()
|
||||
except YouTubeError as exc:
|
||||
log.warning("discovery: own-channel resolve failed (user %s): %s", user.id, exc)
|
||||
# No traceback anywhere in this file's YouTube handlers: for an upstream answer
|
||||
# `_upstream_failure` already logged the structured `youtube.api.error`, and the
|
||||
# message names the call, its status and its reason (or, with no answer at all, the
|
||||
# transport failure). The stack under it is this `except` and the client call.
|
||||
log.warning(
|
||||
"discovery: own-channel resolve failed: %s",
|
||||
exc,
|
||||
extra=logs.event("channels.discovery.own_channel_failed", reason=exc.reason),
|
||||
)
|
||||
|
||||
rows = _discovery_rows(db, user)
|
||||
|
||||
@@ -190,7 +198,11 @@ def discover_channels(
|
||||
rows, key=lambda r: (-int(r[1]), r[0].title is None, (r[0].title or "").lower())
|
||||
)
|
||||
except YouTubeError as exc:
|
||||
log.warning("discovery: channel enrich failed (user %s): %s", user.id, exc)
|
||||
log.warning(
|
||||
"discovery: channel enrich failed: %s",
|
||||
exc,
|
||||
extra=logs.event("channels.discovery.enrich_failed", reason=exc.reason),
|
||||
)
|
||||
db.rollback()
|
||||
|
||||
return [
|
||||
@@ -229,6 +241,7 @@ def _channel_detail_dict(
|
||||
count_live: int,
|
||||
deep_requested: bool,
|
||||
priority: int,
|
||||
hidden: bool,
|
||||
tag_ids: list[int],
|
||||
) -> dict:
|
||||
return {
|
||||
@@ -262,6 +275,9 @@ def _channel_detail_dict(
|
||||
# This user's per-subscription feed priority (0 when not subscribed) so the channel page can
|
||||
# offer the same priority stepper as the manager. Editable only while subscribed.
|
||||
"priority": priority,
|
||||
# This user's per-subscription "hide from feed" flag (False when not subscribed), so the
|
||||
# channel page can offer the same Unhide toggle the manager has. Editable only while subscribed.
|
||||
"hidden": hidden,
|
||||
"backfill_done": channel.backfill_done,
|
||||
"from_explore": channel.from_explore,
|
||||
"details_synced": channel.details_synced_at is not None,
|
||||
@@ -283,9 +299,7 @@ def channel_detail(
|
||||
needed — a stub, or a channel last enriched before those fields existed (published_at is a
|
||||
reliable sentinel: every real channel has a creation date). The enrich uses the API key
|
||||
(1 unit, cached after), so it's skipped for the quota-less demo account."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if channel.published_at is None and not user.is_demo:
|
||||
try:
|
||||
with (
|
||||
@@ -296,7 +310,13 @@ def channel_detail(
|
||||
db.commit()
|
||||
channel = db.get(Channel, channel_id)
|
||||
except YouTubeError as exc:
|
||||
log.warning("channel detail enrich failed (%s): %s", channel_id, exc)
|
||||
log.warning(
|
||||
"channel detail enrich failed: %s",
|
||||
exc,
|
||||
extra=logs.event(
|
||||
"channels.detail.enrich_failed", channel_id=channel_id, reason=exc.reason
|
||||
),
|
||||
)
|
||||
db.rollback()
|
||||
sub = db.execute(
|
||||
select(Subscription).where(
|
||||
@@ -344,6 +364,7 @@ def channel_detail(
|
||||
count_live=count_live,
|
||||
deep_requested=bool(sub.deep_requested) if sub else False,
|
||||
priority=int(sub.priority) if sub else 0,
|
||||
hidden=bool(sub.hidden) if sub else False,
|
||||
tag_ids=tag_ids,
|
||||
)
|
||||
|
||||
@@ -361,16 +382,11 @@ def explore_channel(
|
||||
it, then ingests one page of its uploads (newest-first) marked via_explore + enriched.
|
||||
Returns the next page token so the page can "load more". Guarded against the shared
|
||||
backfill quota reserve; blocked for the demo account (it spends shared quota)."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if _is_blocked(db, user, channel_id):
|
||||
raise HTTPException(status_code=403, detail="You've blocked this channel.")
|
||||
raise refused(403, "channel_blocked")
|
||||
if quota.backfill_reserve_reached():
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="The shared daily quota reserve is reached — try exploring later.",
|
||||
)
|
||||
raise refused(429, "quota_reserve")
|
||||
|
||||
# Record/refresh the exploration (DB only) before ingesting, so the explorer can see the
|
||||
# videos as soon as they land. Flag the channel ephemeral unless it's organically followed.
|
||||
@@ -406,8 +422,9 @@ def explore_channel(
|
||||
channel = db.get(Channel, channel_id)
|
||||
result = explore_ingest_page(db, yt, channel, page_token)
|
||||
except YouTubeError as exc:
|
||||
raise HTTPException(status_code=422, detail=f"YouTube couldn't load this channel: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
return {
|
||||
"ok": True,
|
||||
"channel_id": channel_id,
|
||||
"ingested": result["ingested"],
|
||||
"next_page_token": result["next_page_token"],
|
||||
@@ -421,7 +438,7 @@ def _user_subscription(db: Session, user: User, channel_id: str) -> Subscription
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if sub is None:
|
||||
raise HTTPException(status_code=404, detail="Not subscribed to this channel")
|
||||
raise not_found("subscription")
|
||||
return sub
|
||||
|
||||
|
||||
@@ -489,6 +506,7 @@ def update_channel(
|
||||
background.add_task(backfill_channel_now, channel_id, user.id)
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"id": channel_id,
|
||||
"priority": sub.priority,
|
||||
"hidden": sub.hidden,
|
||||
@@ -507,9 +525,7 @@ def reset_backfill(
|
||||
state (a "reset" trigger). Clears the channel's backfill markers, opts it back into deep
|
||||
backfill, and re-runs its recent pull immediately; the deep scheduler re-pages the full
|
||||
back-catalog on its next run. Idempotent — videos upsert by id, so nothing duplicates."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
channel.backfill_done = False
|
||||
channel.backfill_cursor = None
|
||||
channel.recent_synced_at = None
|
||||
@@ -525,7 +541,7 @@ def reset_backfill(
|
||||
db.commit()
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
||||
run_recent_backfill(db, [channel], max_channels=1)
|
||||
return {"id": channel_id, "reset": True}
|
||||
return {"ok": True, "id": channel_id, "reset": True}
|
||||
|
||||
|
||||
@router.post("/{channel_id}/subscribe")
|
||||
@@ -539,16 +555,14 @@ def subscribe(
|
||||
local subscription with YouTube's returned resource id and enriches the channel's metadata
|
||||
if it's still a stub, so the new row renders properly in the manager. It does NOT pull the
|
||||
channel's videos — the scheduler picks those up on its next run, like any subscribed channel."""
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
existing = db.execute(
|
||||
select(Subscription).where(
|
||||
Subscription.user_id == user.id, Subscription.channel_id == channel_id
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise HTTPException(status_code=409, detail="Already subscribed to this channel.")
|
||||
raise refused(409, "already_subscribed")
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_SUBSCRIBE),
|
||||
YouTubeClient(db, user) as yt,
|
||||
@@ -562,13 +576,14 @@ def subscribe(
|
||||
# subscription resync fills in the resource id. Any other YouTube error is a real
|
||||
# problem — surface it with a clear message (422, not a 502 that the client would
|
||||
# mistake for a transient "connection lost").
|
||||
msg = str(exc).lower()
|
||||
if "already exists" in msg or "duplicate" in msg:
|
||||
# On the reason code, never on the prose: the error no longer carries the body (it is
|
||||
# the request quoted back), and a message YouTube rewords would silently turn "already
|
||||
# following" back into a 422 the user cannot act on. `reason` is comma-joined when the
|
||||
# body listed several.
|
||||
if "subscriptionDuplicate" in exc.reason.split(","):
|
||||
yt_sub_id = ""
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=422, detail=f"YouTube couldn't subscribe you: {exc}"
|
||||
)
|
||||
raise youtube_refusal(exc)
|
||||
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
|
||||
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
|
||||
# render. Keep the client open (we're still inside the `with`); a metadata failure here
|
||||
@@ -577,7 +592,15 @@ def subscribe(
|
||||
try:
|
||||
apply_channel_details(db, yt.get_channels([channel_id]))
|
||||
except YouTubeError as exc:
|
||||
log.warning("subscribe: stub enrich failed (%s): %s", channel_id, exc)
|
||||
log.warning(
|
||||
"subscribe: stub enrich failed: %s",
|
||||
exc,
|
||||
extra=logs.event(
|
||||
"channels.subscribe.enrich_failed",
|
||||
channel_id=channel_id,
|
||||
reason=exc.reason,
|
||||
),
|
||||
)
|
||||
db.add(
|
||||
Subscription(
|
||||
user_id=user.id,
|
||||
@@ -597,7 +620,7 @@ def subscribe(
|
||||
.values(via_explore=False)
|
||||
)
|
||||
db.commit()
|
||||
return {"subscribed": channel_id}
|
||||
return {"ok": True, "subscribed": channel_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/subscription")
|
||||
@@ -611,10 +634,7 @@ def unsubscribe(
|
||||
should Hide instead."""
|
||||
sub = _user_subscription(db, user, channel_id)
|
||||
if not sub.yt_subscription_id:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="No YouTube subscription id on record — sync subscriptions first.",
|
||||
)
|
||||
raise refused(400, "no_subscription_id")
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_UNSUBSCRIBE),
|
||||
@@ -624,10 +644,10 @@ def unsubscribe(
|
||||
except YouTubeError as exc:
|
||||
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
|
||||
# a transient "connection lost" gateway blip.
|
||||
raise HTTPException(status_code=422, detail=f"YouTube couldn't unsubscribe you: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
db.delete(sub)
|
||||
db.commit()
|
||||
return {"unsubscribed": channel_id}
|
||||
return {"ok": True, "unsubscribed": channel_id}
|
||||
|
||||
|
||||
@router.get("/blocked/list")
|
||||
@@ -658,8 +678,7 @@ def block_channel(
|
||||
"""Block a channel for this user: its videos won't appear in their live search (and won't be
|
||||
ingested for them) and are hidden from their feed / explore. Idempotent. The now-hidden
|
||||
un-kept search/explore videos are reclaimed by the discovery-cleanup job in due course."""
|
||||
if db.get(Channel, channel_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
get_or_404(db, Channel, channel_id, what="channel")
|
||||
if not _is_blocked(db, user, channel_id):
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=channel_id))
|
||||
# Stop any active exploration of it so it can be cleaned up.
|
||||
@@ -669,7 +688,7 @@ def block_channel(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"blocked": channel_id}
|
||||
return {"ok": True, "blocked": channel_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/block")
|
||||
@@ -684,7 +703,7 @@ def unblock_channel(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"unblocked": channel_id}
|
||||
return {"ok": True, "unblocked": channel_id}
|
||||
|
||||
|
||||
class AttachTagIn(BaseModel):
|
||||
@@ -701,7 +720,7 @@ def attach_tag(
|
||||
_user_subscription(db, user, channel_id)
|
||||
# Only the user's own tags may be attached as personal links (owned_or_404 → 404 if the tag
|
||||
# is missing OR belongs to someone else, without leaking which).
|
||||
owned_or_404(db, Tag, body.tag_id, user, detail="Unknown tag")
|
||||
owned_or_404(db, Tag, body.tag_id, user, what="tag")
|
||||
exists = db.execute(
|
||||
select(ChannelTag).where(
|
||||
ChannelTag.channel_id == channel_id,
|
||||
@@ -719,7 +738,7 @@ def attach_tag(
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"channel_id": channel_id, "tag_id": body.tag_id}
|
||||
return {"ok": True, "channel_id": channel_id, "tag_id": body.tag_id}
|
||||
|
||||
|
||||
@router.delete("/{channel_id}/tags/{tag_id}")
|
||||
@@ -739,4 +758,4 @@ def detach_tag(
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"channel_id": channel_id, "tag_id": tag_id}
|
||||
return {"ok": True, "channel_id": channel_id, "tag_id": tag_id}
|
||||
|
||||
@@ -4,17 +4,23 @@ app.sysconfig. Secret values (e.g. SMTP password) are write-only — never retur
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import mailtext
|
||||
from app import sysconfig
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import User
|
||||
from app.refusal import refused
|
||||
from app.issues import gitea
|
||||
from app.routes._common import issue_refused, not_found
|
||||
from app.refusal import setting_refused
|
||||
|
||||
router = APIRouter(prefix="/api/admin/config", tags=["admin"])
|
||||
|
||||
@@ -28,6 +34,14 @@ def _redact(v):
|
||||
return _URL_USERINFO.sub(r"\1***@", v) if isinstance(v, str) else v
|
||||
|
||||
|
||||
def _apply_runtime(db: Session, key: str) -> None:
|
||||
"""Some knobs change how THIS process behaves, not just what a later read returns — they have to
|
||||
take effect now, without a restart. One dispatch for both call sites (set and reset), so the next
|
||||
such key can't be wired into one of them and forgotten in the other."""
|
||||
if key == "log_level":
|
||||
logs.set_level(sysconfig.get_str(db, "log_level"))
|
||||
|
||||
|
||||
@router.get("")
|
||||
def get_config(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
return sysconfig.describe(db)
|
||||
@@ -49,19 +63,16 @@ def set_config(
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown config key")
|
||||
raise not_found("config_key")
|
||||
if "value" not in body.model_fields_set:
|
||||
raise HTTPException(status_code=400, detail="Missing 'value'")
|
||||
raise refused(400, "value_missing")
|
||||
if s.secret and not sysconfig.secrets_manageable():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Set TOKEN_ENCRYPTION_KEY to store secrets in the database.",
|
||||
)
|
||||
raise refused(400, "no_encryption_key")
|
||||
old = None if s.secret else sysconfig.get(db, key)
|
||||
try:
|
||||
sysconfig.set_value(db, key, body.value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc) or "Invalid value")
|
||||
except sysconfig.InvalidSetting as exc:
|
||||
raise setting_refused(exc) from None
|
||||
# Never log a secret's value — only that it was changed, by whom. Non-secret values are logged
|
||||
# but with URL credentials redacted (a proxy/URL setting can embed user:pass). Skip the row when
|
||||
# a non-secret value didn't actually change (a no-op re-save shouldn't add trail noise); secrets
|
||||
@@ -79,6 +90,7 @@ def set_config(
|
||||
after=None if s.secret else {"value": _redact(new)},
|
||||
)
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
|
||||
@@ -90,7 +102,7 @@ def reset_config(
|
||||
) -> dict:
|
||||
s = sysconfig.spec(key)
|
||||
if s is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown config key")
|
||||
raise not_found("config_key")
|
||||
old = None if s.secret else sysconfig.get(db, key)
|
||||
sysconfig.reset(db, key)
|
||||
audit.record(
|
||||
@@ -103,6 +115,7 @@ def reset_config(
|
||||
before=None if s.secret else {"value": _redact(old)},
|
||||
)
|
||||
db.commit()
|
||||
_apply_runtime(db, key)
|
||||
return sysconfig.describe(db)
|
||||
|
||||
|
||||
@@ -114,8 +127,21 @@ def send_test_email(
|
||||
"""Send a test email to the requesting admin using the current (DB or env) SMTP config —
|
||||
lets the admin verify the settings actually work."""
|
||||
if not email_mod.email_enabled():
|
||||
raise HTTPException(status_code=400, detail="SMTP is not configured.")
|
||||
sent = email_mod.send_test(user.email)
|
||||
raise refused(400, "smtp_off")
|
||||
sent = email_mod.send_test(user.email, mailtext.lang_for_user(user))
|
||||
if not sent:
|
||||
raise HTTPException(status_code=422, detail="SMTP send failed — check the settings.")
|
||||
return {"sent": True, "to": user.email}
|
||||
raise refused(422, "smtp_failed")
|
||||
return {"ok": True, "sent": True, "to": user.email}
|
||||
|
||||
|
||||
@router.post("/test-issues")
|
||||
def test_issue_tracker(
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Check the configured issue tracker (E16): the repository answers with this token, takes
|
||||
issues, and is PRIVATE — the one it must be, since an issue carries its reporter's log."""
|
||||
try:
|
||||
return {"ok": True, **gitea.check(db)}
|
||||
except gitea.GiteaError as exc:
|
||||
raise issue_refused(exc) from None
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, the screenshots, and the evidence under review.
|
||||
|
||||
The session itself — what tags a request, its cap, its level — is `app/debugsession.py`. These routes
|
||||
are the user's handle on their OWN sessions only: another user's id 404s (`owned_or_404`), including
|
||||
for an admin, who reads everyone's lines in the diagnostics console instead. The shared demo account
|
||||
cannot record: its log would interleave every visitor under one account.
|
||||
|
||||
Screenshots arrive as a raw image body (`image/png`, `image/jpeg`, `image/webp`), not a multipart
|
||||
form — one image per request, capped while it arrives, and kept only if its bytes really are one of
|
||||
those types, whatever the Content-Type claimed.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, debugsession, logs, logsink
|
||||
from app.audit import AuditAction
|
||||
from app.auth import require_human
|
||||
from app.db import get_db
|
||||
from app.issues import bundle, gitea
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.refusal import refused
|
||||
from app.models import DebugScreenshot, DebugSession, User
|
||||
from app.routes._common import capped_body, issue_refused, not_found, owned_or_404
|
||||
from app.utils import iso
|
||||
|
||||
router = APIRouter(prefix="/api/debug-sessions", tags=["diagnostics"])
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# Sessions listed in Settings → Diagnosztika, newest first: enough to reach last week's unsent draft.
|
||||
LIST_MAX = 20
|
||||
|
||||
|
||||
class StartIn(BaseModel):
|
||||
level: debugsession.SessionLevel
|
||||
|
||||
|
||||
_view = debugsession.view
|
||||
|
||||
# New sessions a user may open per hour (joining the running one is not counted): a session is a
|
||||
# place to keep screenshots, and this — with the per-user storage cap — bounds how many there are.
|
||||
_new_sessions = RateLimiter(max_events=20, window_seconds=3600)
|
||||
|
||||
|
||||
@router.post("")
|
||||
def start_session(
|
||||
body: StartIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Start recording, or join the recording already running (another tab started it)."""
|
||||
joining = debugsession.open_session(db, user) is not None
|
||||
if not joining and not _new_sessions.allow(str(user.id)):
|
||||
raise refused(429, "debug_session_rate_limited", limit=_new_sessions.max_events)
|
||||
try:
|
||||
session = debugsession.start(db, user, body.level)
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
# A concurrent Start won the one-open-per-user index: join that one.
|
||||
db.rollback()
|
||||
session = debugsession.open_session(db, user)
|
||||
if session is None:
|
||||
raise refused(409, "debug_session_conflict") from None
|
||||
# The rest of THIS request is part of the session too.
|
||||
level, until = debugsession.level_for(user, session)
|
||||
logs.set_user(user.id, level, until, session.id)
|
||||
log.info(
|
||||
"debug session started",
|
||||
extra=logs.event("debug.session.started", level=session.level, until=iso(session.until)),
|
||||
)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.post("/{sid}/stop")
|
||||
def stop_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if session.ended_at is None:
|
||||
# Logged BEFORE the session closes, while this request still carries its tag: the last line
|
||||
# of the evidence is the Stop itself.
|
||||
log.info("debug session stopped", extra=logs.event("debug.session.stopped"))
|
||||
debugsession.stop(session)
|
||||
db.commit()
|
||||
logs.end_session_elevation(session.id)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_sessions(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
sessions = db.scalars(
|
||||
select(DebugSession)
|
||||
.where(DebugSession.user_id == user.id)
|
||||
.order_by(DebugSession.started_at.desc())
|
||||
.limit(LIST_MAX)
|
||||
).all()
|
||||
return {"items": [_view(db, s) for s in sessions]}
|
||||
|
||||
|
||||
@router.get("/{sid}")
|
||||
def get_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
return _view(db, owned_or_404(db, DebugSession, sid, user, what="debug_session"))
|
||||
|
||||
|
||||
@router.get("/{sid}/events")
|
||||
def session_events(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""The rows an issue from this session would carry — browser and server, in time order."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
rows, total = debugsession.bundle(db, session)
|
||||
return {"items": [logsink.serialize(r) for r in rows], "total": total}
|
||||
|
||||
|
||||
# --- the draft and its export ---------------------------------------------------------------
|
||||
# The composer's own fields (steps, description…) are its business, kept whole so it reopens as it
|
||||
# was — bounded here so a draft cannot become a place to store anything else.
|
||||
FORM_MAX_BYTES = 64 * 1024
|
||||
# Zips a user may build per minute: each holds every screenshot in memory while it is built.
|
||||
_exports = RateLimiter(max_events=6, window_seconds=60)
|
||||
|
||||
|
||||
class DraftIn(BaseModel):
|
||||
title: str = Field(max_length=200)
|
||||
# The rendered markdown the export and the issue carry.
|
||||
body: str = Field(max_length=50_000)
|
||||
# Log rows taken out of the evidence in the review.
|
||||
excluded: list[int] = Field(default_factory=list, max_length=debugsession.BUNDLE_MAX)
|
||||
form: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
@router.put("/{sid}/draft")
|
||||
def save_draft(
|
||||
sid: str,
|
||||
body: DraftIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if len(json.dumps(body.form, ensure_ascii=False).encode()) > FORM_MAX_BYTES:
|
||||
raise HTTPException(status_code=413, detail={"code": "too_large"})
|
||||
# Under the lock, and refused while a send is under way: the issue is being built from the draft
|
||||
# as it was saved, and this would change it under the send.
|
||||
debugsession.lock(db, session)
|
||||
if debugsession.sending(session):
|
||||
raise refused(409, "issue_sending")
|
||||
if session.issue_url is not None:
|
||||
# Sent: the issue is on the tracker now, and a draft edited after it would say otherwise.
|
||||
raise refused(409, "issue_already_sent")
|
||||
debugsession.save_draft(session, body.model_dump())
|
||||
db.commit()
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
@router.get("/{sid}/export")
|
||||
def export_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
"""The issue as a zip — the text, the two logs and the screenshots — for an instance without an
|
||||
issue tracker, or for sending it by hand. Built from the SAVED draft: what the composer shows is
|
||||
what it saved."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if not _exports.allow(str(user.id)):
|
||||
raise refused(429, "export_rate_limited")
|
||||
data = bundle.zipped(bundle.files(db, session))
|
||||
name = f"siftlode-debug-{session.started_at:%Y%m%d-%H%M}-{session.id[:6]}.zip"
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="application/zip",
|
||||
headers={
|
||||
"Content-Disposition": f'attachment; filename="{name}"',
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# --- the issue tracker (S4) -----------------------------------------------------------------
|
||||
# Issues a user may open per hour — counted once the tracker is ASKED to open one (an answer that
|
||||
# goes missing may still have created it, so an attempt cannot be given back); finishing a half-sent
|
||||
# one, or an attempt refused before the tracker was asked, is not counted. The composer sends one at
|
||||
# a time; this is room for a bad afternoon, not a pace anything legitimate keeps up.
|
||||
_new_issues = RateLimiter(max_events=5, window_seconds=3600)
|
||||
# Attempts on the tracker per hour, creating or finishing. Deliberately NOT the budget above: a
|
||||
# half-sent issue must always be finishable, so the pace that guards creating one must never be what
|
||||
# stands between a user and their own report. This one only bounds a loop — every attempt re-reads
|
||||
# the whole bundle into memory and re-offers the pending file to the tracker, and a retry that is
|
||||
# refused for good (the tracker's own limits) never leaves `pending`. No hand reaches it.
|
||||
_sends = RateLimiter(max_events=60, window_seconds=3600)
|
||||
# The composer's live preview, rendered BY the tracker: one authenticated call each, with our token.
|
||||
# The composer's toggle fires it once per switch INTO preview — the editor is the other branch, so
|
||||
# no keystroke can change the query key while the preview is up — and react-query's cache makes an
|
||||
# unchanged draft free. (It does NOT debounce, whatever this comment used to say: a key that changes
|
||||
# per keystroke is one fetch per keystroke, measured. Showing the preview BESIDE the fields would
|
||||
# need a real debounce first.) This bounds what a loop can aim at the tracker.
|
||||
_previews = RateLimiter(max_events=60, window_seconds=60)
|
||||
|
||||
|
||||
def _issues_rate_limited() -> HTTPException:
|
||||
return refused(429, "issues_rate_limited", limit=_new_issues.max_events)
|
||||
|
||||
|
||||
@router.post("/{sid}/issue")
|
||||
def send_issue(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Send the session's issue to the tracker — from the SAVED draft, as the export is built — or
|
||||
finish a send that stopped half-way (the issue exists; only what is missing is sent)."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
# The session is CLAIMED for the send rather than locked through it: the tracker is slow (a
|
||||
# dozen calls, 20 s each), and a row lock held across them would queue this user's Stop, their
|
||||
# autosave and their screenshots behind it. They see the claim instead and are refused at once.
|
||||
debugsession.lock(db, session)
|
||||
if debugsession.sending(session):
|
||||
raise refused(409, "issue_sending")
|
||||
new = session.issue_url is None
|
||||
if not new and not debugsession.unfinished(session):
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
key = str(user.id)
|
||||
if new:
|
||||
if not ((session.draft or {}).get("title") or "").strip():
|
||||
raise refused(422, "issue_title_required")
|
||||
if _new_issues.exhausted(key):
|
||||
raise _issues_rate_limited()
|
||||
if not _sends.allow(key):
|
||||
raise refused(429, "issue_rate_limited", limit=_sends.max_events)
|
||||
token = debugsession.claim_send(session)
|
||||
db.commit() # the claim is what everything else sees; the lock ends here
|
||||
|
||||
def count_new_issue() -> None:
|
||||
if not _new_issues.allow(key):
|
||||
raise _issues_rate_limited()
|
||||
|
||||
try:
|
||||
gitea.deliver(
|
||||
db,
|
||||
session,
|
||||
token=token,
|
||||
checkpoint=db.commit,
|
||||
before_create=count_new_issue,
|
||||
on_created=lambda: _record_created(db, user, session),
|
||||
)
|
||||
except debugsession.SendClaimLost:
|
||||
# This send outlived its lease and another one took the session over while it was still
|
||||
# talking to the tracker. It stopped rather than writing over that one's progress — and a
|
||||
# send IS under way, just not ours, which is what the caller needs to hear.
|
||||
raise refused(409, "issue_sending") from None
|
||||
except gitea.GiteaError as exc:
|
||||
# Keep what did happen: an issue created before an upload failed stays linked, and the
|
||||
# retry does only what is still missing.
|
||||
raise issue_refused(exc) from None
|
||||
except HTTPException:
|
||||
raise # the pace, refused before the tracker was asked
|
||||
finally:
|
||||
_end_send(db, session, token)
|
||||
return {"ok": True, "session": _view(db, session)}
|
||||
|
||||
|
||||
def _end_send(db: Session, session: DebugSession, token: str) -> None:
|
||||
"""Give the claim back, whatever else happened. A failure here must not replace the failure that
|
||||
brought us: a dead transaction (the database went away mid-send) is rolled back, and the claim
|
||||
then simply ages out."""
|
||||
try:
|
||||
debugsession.lock(db, session)
|
||||
debugsession.release_send(session, token)
|
||||
db.commit()
|
||||
except Exception: # noqa: BLE001 — whatever went wrong here, the send's own failure is the news
|
||||
db.rollback()
|
||||
log.warning(
|
||||
"a send could not give its claim back",
|
||||
extra=logs.event("issues.claim.stuck", session_id=session.id),
|
||||
)
|
||||
|
||||
|
||||
def _record_created(db: Session, user: User, session: DebugSession) -> None:
|
||||
"""The trail for the one act that takes a user's log off this instance. Staged inside the send,
|
||||
so the checkpoint that first writes `issue_url` commits BOTH — recorded later it could be lost
|
||||
on its own (the process dies mid-send), and no retry could write it afterwards: the next send
|
||||
sees the link and knows the issue is not new."""
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
AuditAction.ISSUE_CREATE,
|
||||
target_type="debug_session",
|
||||
target_id=session.id,
|
||||
summary=session.issue_url or "",
|
||||
)
|
||||
|
||||
|
||||
class PreviewIn(BaseModel):
|
||||
text: str = Field(max_length=50_000)
|
||||
|
||||
|
||||
@router.post("/{sid}/preview")
|
||||
def preview_issue(
|
||||
sid: str,
|
||||
body: PreviewIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The body as the tracker will render it. Gitea sanitises its own output; the composer still
|
||||
shows it in a sandboxed frame, never in the app's own document."""
|
||||
owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if not _previews.allow(str(user.id)):
|
||||
raise refused(429, "preview_rate_limited")
|
||||
try:
|
||||
return {"ok": True, "html": gitea.preview(db, body.text)}
|
||||
except gitea.GiteaError as exc:
|
||||
raise issue_refused(exc) from None
|
||||
|
||||
|
||||
# --- screenshots ---------------------------------------------------------------------------
|
||||
def _shot_view(shot: DebugScreenshot) -> dict:
|
||||
return {"id": shot.id, "mime": shot.mime, "size": shot.size, "created_at": iso(shot.created_at)}
|
||||
|
||||
|
||||
@router.get("/{sid}/screenshots")
|
||||
def list_screenshots(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
return {"items": [_shot_view(s) for s in debugsession.screenshots(db, session)]}
|
||||
|
||||
|
||||
def _upload_target(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> DebugSession:
|
||||
return owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
|
||||
|
||||
async def _image_body(request: Request) -> bytes:
|
||||
return await capped_body(request, debugsession.SCREENSHOT_MAX_BYTES)
|
||||
|
||||
|
||||
@router.post("/{sid}/screenshots")
|
||||
def add_screenshot(
|
||||
# The owner check FIRST: dependencies resolve in order, so a stranger's upload is refused
|
||||
# before a byte of it is read.
|
||||
session: DebugSession = Depends(_upload_target),
|
||||
data: bytes = Depends(_image_body),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
try:
|
||||
shot = debugsession.add_screenshot(db, session, data)
|
||||
except debugsession.ScreenshotRefused as exc:
|
||||
raise _screenshot_refused(exc.code) from None
|
||||
db.commit()
|
||||
return {"ok": True, "screenshot": _shot_view(shot)}
|
||||
|
||||
|
||||
def _screenshot_refused(code: str) -> HTTPException:
|
||||
# Literal reasons, one per case: the frontend's contract test reads them to require a sentence
|
||||
# for each in both languages.
|
||||
if code == "already_sent":
|
||||
return refused(409, "screenshot_already_sent")
|
||||
if code == "too_many":
|
||||
return refused(422, "screenshot_too_many", limit=debugsession.SCREENSHOTS_MAX)
|
||||
if code == "sending":
|
||||
return refused(409, "issue_sending")
|
||||
if code == "storage_full":
|
||||
mb = debugsession.SCREENSHOT_USER_MAX_BYTES // (1024 * 1024)
|
||||
return refused(422, "screenshot_storage_full", limit=mb)
|
||||
if code == "empty":
|
||||
return refused(422, "screenshot_empty")
|
||||
return refused(422, "screenshot_not_an_image")
|
||||
|
||||
|
||||
def _session_screenshot(db: Session, sid: str, shot_id: int, user: User) -> DebugScreenshot:
|
||||
return _screenshot_of(
|
||||
db, owned_or_404(db, DebugSession, sid, user, what="debug_session"), shot_id
|
||||
)
|
||||
|
||||
|
||||
def _screenshot_of(db: Session, session: DebugSession, shot_id: int) -> DebugScreenshot:
|
||||
shot = db.get(DebugScreenshot, shot_id)
|
||||
if shot is None or shot.session_id != session.id:
|
||||
raise not_found("screenshot")
|
||||
return shot
|
||||
|
||||
|
||||
@router.get("/{sid}/screenshots/{shot_id}")
|
||||
def get_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
shot = _session_screenshot(db, sid, shot_id, user)
|
||||
# `nosniff`: the type was checked against the bytes at upload, and the browser must not guess.
|
||||
return Response(
|
||||
content=shot.data,
|
||||
media_type=shot.mime,
|
||||
headers={"Cache-Control": "private, max-age=3600", "X-Content-Type-Options": "nosniff"},
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{sid}/screenshots/{shot_id}")
|
||||
def delete_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
shot = _screenshot_of(db, session, shot_id)
|
||||
try:
|
||||
debugsession.remove_screenshot(db, session, shot)
|
||||
except debugsession.ScreenshotRefused as exc:
|
||||
raise _screenshot_refused(exc.code) from None
|
||||
db.commit()
|
||||
return {"ok": True, "deleted": shot_id}
|
||||
@@ -0,0 +1,186 @@
|
||||
"""The browser's log, sent while its user has a raised diagnostics level (E22 S5).
|
||||
|
||||
The client keeps its log in memory (frontend `lib/log.ts`). Only after the user consented to Részletes
|
||||
or Teljes in Settings → Diagnosztika does it stream that log here, in batches, and each batch lands in
|
||||
`log_events` beside the server's own rows: `source="web"`, the account from the SESSION (never from
|
||||
the body), and the request id each event carries, which is what joins a browser line to the server
|
||||
line of the same request.
|
||||
|
||||
Nothing from the body is trusted as-is:
|
||||
* only a JSON Content-Type is read, and the whole body is capped before it is parsed, and so are
|
||||
the number of events and each event's attributes;
|
||||
* the pace is capped per account, and so is what an account may store per hour;
|
||||
* everything is redacted AGAIN with the server's own rules — the client redacts at capture, but a
|
||||
modified client would not;
|
||||
* an event below the consented level is dropped (a Részletes consent does not store what only
|
||||
Teljes may carry), and so is the whole batch once the level has lapsed — where an open debug
|
||||
session the request names counts as consent to ITS level (E16), and only its own events keep
|
||||
its tag;
|
||||
* the timestamp is clamped to the consent window, so a row cannot dodge the retention job by
|
||||
claiming to come from the future.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from sqlalchemy import insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import debugsession, diagnostics, logs, logsink
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import DebugSession, LogEvent, User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.routes._common import capped_body
|
||||
|
||||
router = APIRouter(prefix="/api/diagnostics", tags=["diagnostics"])
|
||||
|
||||
BODY_MAX_BYTES = 512 * 1024
|
||||
EVENTS_MAX = 200
|
||||
ATTRS_MAX_BYTES = 16 * 1024
|
||||
# The client flushes every few seconds and when its buffer fills; this is room for a burst, not a
|
||||
# pace anything legitimate keeps up for a minute.
|
||||
_batches = RateLimiter(max_events=30, window_seconds=60)
|
||||
# What one account may STORE per hour. The pace cap alone still lets a self-consented account write
|
||||
# ~15 MiB a minute into the one shared database, kept for the whole retention window; a real session
|
||||
# at Teljes stores a few hundred kB an hour. Past the budget a batch is accepted and not stored (a
|
||||
# refusal would only make the client resend it), and one notice per hour says so.
|
||||
STORED_BYTES_PER_HOUR = 5 * 1024 * 1024
|
||||
_stored_bytes = RateLimiter(max_events=STORED_BYTES_PER_HOUR, window_seconds=3600)
|
||||
_budget_notices = RateLimiter(max_events=1, window_seconds=3600)
|
||||
|
||||
ClientLevel = Literal["TRACE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
|
||||
|
||||
|
||||
class ClientEvent(BaseModel):
|
||||
ts: datetime
|
||||
level: ClientLevel
|
||||
event: str = Field(max_length=64, pattern=r"^[a-z][a-z0-9_]*(\.[a-z0-9_]+)*$")
|
||||
msg: str = Field(max_length=8_000)
|
||||
attrs: dict[str, Any] | None = None
|
||||
request_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$")
|
||||
tab_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,32}$")
|
||||
# The debug session the event was captured in (E16); kept only when it is the one this request
|
||||
# names in `X-Debug-Session` and the auth dependency found it the caller's own and open.
|
||||
debug_session_id: str | None = Field(default=None, pattern=r"^[0-9a-f]{16}$")
|
||||
version: str | None = Field(default=None, max_length=64)
|
||||
|
||||
|
||||
class ClientEventsIn(BaseModel):
|
||||
events: list[ClientEvent] = Field(max_length=EVENTS_MAX)
|
||||
|
||||
|
||||
async def _parsed_events(request: Request) -> ClientEventsIn:
|
||||
"""The body as the model — parsed only after the size cap, which is why this is a dependency and
|
||||
not a plain body parameter (FastAPI reads a declared body in full before anything else runs).
|
||||
|
||||
JSON only, by its Content-Type, as a declared body would be. Parsing any body as JSON would let a
|
||||
CORS-simple `text/plain` form post from a same-site page write events into a signed-in user's
|
||||
log; the app's own client always sends `application/json`, the unload beacon included."""
|
||||
media_type = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if media_type != "application/json" and not media_type.endswith("+json"):
|
||||
raise HTTPException(status_code=415, detail={"code": "json_required"})
|
||||
raw = await capped_body(request, BODY_MAX_BYTES)
|
||||
try:
|
||||
return ClientEventsIn.model_validate_json(raw)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=422, detail=json.loads(exc.json())) from exc
|
||||
|
||||
|
||||
def _row(
|
||||
item: ClientEvent,
|
||||
user: User,
|
||||
now: datetime,
|
||||
window_start: datetime,
|
||||
session: DebugSession | None,
|
||||
) -> dict[str, Any]:
|
||||
attrs = logs.redact(item.attrs) if item.attrs else None
|
||||
if attrs is not None:
|
||||
size = len(json.dumps(attrs, ensure_ascii=False, default=str).encode())
|
||||
if size > ATTRS_MAX_BYTES:
|
||||
attrs = {"attrs_dropped_bytes": size}
|
||||
if item.tab_id:
|
||||
attrs = {**(attrs or {}), "tab_id": item.tab_id}
|
||||
ts = item.ts if item.ts.tzinfo else item.ts.replace(tzinfo=timezone.utc)
|
||||
return logsink.to_row(
|
||||
{
|
||||
"ts": min(max(ts, window_start), now),
|
||||
"level": item.level,
|
||||
"sev": logs.SEVERITY[item.level],
|
||||
"source": "web",
|
||||
"logger": "web",
|
||||
"event": item.event,
|
||||
"msg": logs.redact_text(item.msg),
|
||||
"attrs": attrs,
|
||||
"request_id": item.request_id,
|
||||
"user_id": user.id,
|
||||
"debug_session_id": (
|
||||
session.id if session and item.debug_session_id == session.id else None
|
||||
),
|
||||
"version": item.version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.post("/events")
|
||||
def ingest_events(
|
||||
# The account first: dependencies resolve in order, so an anonymous caller is refused before a
|
||||
# byte of its body is read.
|
||||
request: Request,
|
||||
user: User = Depends(current_user),
|
||||
body: ClientEventsIn = Depends(_parsed_events),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# A debug session the request names raises the level for its own events (E16): starting one is
|
||||
# the consent to that level for its duration, whatever the user's own level is.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, _ = debugsession.level_for(user, session)
|
||||
if level == diagnostics.BASE:
|
||||
# Lapsed or withdrawn: the client stops streaming when it sees this.
|
||||
raise HTTPException(status_code=409, detail={"code": "diagnostics_off"})
|
||||
if not _batches.allow(str(user.id)):
|
||||
raise HTTPException(status_code=429, detail={"code": "rate_limited"})
|
||||
now = datetime.now(timezone.utc)
|
||||
window_start = now - timedelta(hours=diagnostics.ELEVATION_HOURS)
|
||||
floor = logs.SEVERITY[level]
|
||||
rows = [
|
||||
_row(item, user, now, window_start, session)
|
||||
for item in body.events
|
||||
if logs.SEVERITY[item.level] >= floor
|
||||
]
|
||||
if not rows:
|
||||
return {"ok": True, "stored": 0}
|
||||
size = sum(len(json.dumps(row, ensure_ascii=False, default=str).encode()) for row in rows)
|
||||
stored = len(rows)
|
||||
if not _stored_bytes.allow(str(user.id), size):
|
||||
stored = 0
|
||||
if not _budget_notices.allow(str(user.id)):
|
||||
return {"ok": True, "stored": 0}
|
||||
rows = [_budget_notice(user, now)]
|
||||
# Under the erasure guard until the commit: an account erased meanwhile gets no rows back.
|
||||
with logs.erasure_guard():
|
||||
db.execute(insert(LogEvent), logsink.without_erased(rows))
|
||||
db.commit()
|
||||
return {"ok": True, "stored": stored}
|
||||
|
||||
|
||||
def _budget_notice(user: User, now: datetime) -> dict[str, Any]:
|
||||
return logsink.to_row(
|
||||
{
|
||||
"ts": now,
|
||||
"level": "WARNING",
|
||||
"sev": logs.SEVERITY["WARNING"],
|
||||
"source": "web",
|
||||
"logger": "web",
|
||||
"event": "diagnostics.budget_spent",
|
||||
# What the budget does, exactly: it is a SLIDING hour, and a refused batch is not
|
||||
# charged to it, so smaller batches keep landing while there is room for them. Saying
|
||||
# "the rest of the hour is not kept" would be a row contradicting the rows beneath it.
|
||||
"msg": "The browser log passed its hourly storage budget; a batch bigger than what is left of it is not kept",
|
||||
"attrs": {"budget_bytes": STORED_BYTES_PER_HOUR},
|
||||
"user_id": user.id,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,192 @@
|
||||
"""E24 · the Home surface's HTTP shell.
|
||||
|
||||
Thin by construction: every decision — what counts as in progress, what fits a time budget, which
|
||||
tier the hero comes from — lives in `app/discover/`, so the scheduler, a future CLI or a test can ask
|
||||
the same questions without a request. This file validates, gates and serializes.
|
||||
|
||||
The consolidated `GET /api/discover` mirrors the Plex Discover route's one-payload shape (E5 S4): a
|
||||
landing page that fires four requests to answer one question would spend its whole budget on the
|
||||
thing it exists to remove.
|
||||
"""
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user, require_human
|
||||
from app.db import get_db
|
||||
from app.discover import hero as hero_mod
|
||||
from app.discover import pins, rails
|
||||
from app.discover.budget import Budget, DEFAULT_BUDGET
|
||||
from app.models import ComfortPin, User
|
||||
from app.routes._common import owned_or_404, not_found
|
||||
|
||||
router = APIRouter(prefix="/api/discover", tags=["discover"])
|
||||
|
||||
|
||||
class PinIn(BaseModel):
|
||||
# From `pins.PIN_KINDS`, not a second hand-written list: the vocabulary, the schema columns and
|
||||
# this request contract have to move together, and re-typing them here is how they would not.
|
||||
kind: Literal[*pins.PIN_KINDS]
|
||||
# The id the rest of the API speaks for that kind: a video id, a channel id, a Plex rating_key,
|
||||
# or a playlist id. `pins.resolve_ref` does the translation to the row's internal key.
|
||||
ref: str
|
||||
|
||||
|
||||
class ReorderIn(BaseModel):
|
||||
ids: list[int]
|
||||
|
||||
|
||||
@router.get("")
|
||||
def get_discover(
|
||||
budget: Budget = DEFAULT_BUDGET,
|
||||
seed: int = 0,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Everything the Home page renders, in one pass.
|
||||
|
||||
`resume` and `shelf` are built once and handed to the hero, so the two rails the hero draws from
|
||||
are not queried twice for a single page load."""
|
||||
resume = rails.continue_watching(db, user)
|
||||
# Plex off → the shelf rail (and so the hero's comfort tier) goes YouTube-only like the rails
|
||||
# beside it; the pins are kept and `GET /pins` still lists them (`pins.shelf`).
|
||||
shelf = pins.shelf(db, user, include_plex=rails.plex_available(db))
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(db, user, budget, seed, resume=resume, shelf=shelf),
|
||||
"continue_watching": resume,
|
||||
"comfort": shelf,
|
||||
}
|
||||
|
||||
|
||||
# How many already-shown cards a reroll may carry, matched to the client's own rolling window
|
||||
# (`SEEN_WINDOW` in `Home.tsx`). A cheap sanity clamp, and deliberately no more than that: `exclude`
|
||||
# never reaches SQL — `hero.pick` uses it only as set membership over an already-bounded candidate
|
||||
# pool (the resume rail's `RAIL_LIMIT` rows, the shelf, `SURPRISE_DRAW` surprise draws) — so there is
|
||||
# no scan for a crafted URL to unbound, and the set is built in full before this slice anyway. The
|
||||
# comment that claimed otherwise was corrected in round 2; the request line's own length is what
|
||||
# actually bounds a hostile `exclude`.
|
||||
MAX_EXCLUDE = 32
|
||||
|
||||
|
||||
@router.get("/pick")
|
||||
def get_pick(
|
||||
budget: Budget = DEFAULT_BUDGET,
|
||||
seed: int = 0,
|
||||
exclude: str = "",
|
||||
start: int = 0,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The "Mást kérek" reroll: a hero on its own, for a new seed and/or a new budget.
|
||||
|
||||
One seed drives BOTH sources (see `discover/queries.py`) — otherwise a reroll would re-draw the
|
||||
Plex half while the YouTube half stood still, and the button would look broken half the time.
|
||||
|
||||
`exclude` is a comma-separated list of `<source>:<id>` keys the user has already been shown. It
|
||||
is what makes the button honest: a seed alone only moves the surprise tier, so a user with
|
||||
something half-finished got the same card back every press.
|
||||
|
||||
`start` is the press count: it rotates which tier answers first, so consecutive presses offer a
|
||||
different KIND of answer instead of walking down the resume rail. Taken modulo the tier count in
|
||||
`hero.pick`, so any integer is valid and no value can 422 a button."""
|
||||
skip = {p for p in exclude.split(",") if p}
|
||||
return {
|
||||
"budget": budget,
|
||||
"hero": hero_mod.pick(
|
||||
db, user, budget, seed, exclude=set(list(skip)[:MAX_EXCLUDE]), start=start
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/pins")
|
||||
def list_pins(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The shelf on its own — for the shelf editor, which needs it without the rest of the page."""
|
||||
return {"items": pins.shelf(db, user)}
|
||||
|
||||
|
||||
@router.post("/pins")
|
||||
def add_pin(
|
||||
body: PinIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Pin something to the comfort shelf.
|
||||
|
||||
`require_human`, like every other personal-state write: the shared demo account must not be able
|
||||
to curate a communal shelf. Re-pinning something already on the shelf is a no-op that returns the
|
||||
existing pin rather than a 409 — the gesture is "make sure this is on my shelf", and the unique
|
||||
constraint is the mechanism, not the message."""
|
||||
resolved = pins.resolve_ref(db, user, body.kind, body.ref)
|
||||
if resolved is None:
|
||||
raise not_found("item")
|
||||
column, value = resolved
|
||||
|
||||
existing = db.execute(
|
||||
select(ComfortPin).where(
|
||||
ComfortPin.user_id == user.id,
|
||||
getattr(ComfortPin, column) == value,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return {"ok": True, "id": existing.id, "created": False}
|
||||
|
||||
pin = ComfortPin(user_id=user.id, position=pins.next_position(db, user), **{column: value})
|
||||
db.add(pin)
|
||||
try:
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
# A concurrent pin of the same thing from a second tab. The unique constraint did its job;
|
||||
# the user's intent ("be on my shelf") is satisfied either way, so report the winner rather
|
||||
# than a 500 the user can do nothing with.
|
||||
db.rollback()
|
||||
winner = db.execute(
|
||||
select(ComfortPin).where(
|
||||
ComfortPin.user_id == user.id,
|
||||
getattr(ComfortPin, column) == value,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if winner is None:
|
||||
# Not the unique constraint, then: the only other way in is the target row being pruned
|
||||
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
||||
# item), which trips the FK. Semantically that is the same "no such thing" the pre-check
|
||||
# returns above — re-raising handed the user a 500 they could do nothing with.
|
||||
raise not_found("item")
|
||||
return {"ok": True, "id": winner.id, "created": False}
|
||||
return {"ok": True, "id": pin.id, "created": True}
|
||||
|
||||
|
||||
@router.delete("/pins/{pin_id}")
|
||||
def remove_pin(
|
||||
pin_id: int,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pin = owned_or_404(db, ComfortPin, pin_id, user, what="pin")
|
||||
db.delete(pin)
|
||||
db.commit()
|
||||
return {"deleted": pin_id}
|
||||
|
||||
|
||||
@router.post("/pins/reorder")
|
||||
def reorder_pins(
|
||||
body: ReorderIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Rewrite the shelf order from the list of pin ids the client shows.
|
||||
|
||||
Whole-list rather than a move-one delta: the shelf is short, and a full rewrite cannot leave two
|
||||
tiles claiming the same slot the way an interrupted sequence of single moves can. The list may be
|
||||
PARTIAL (hidden Plex pins, a stale tab) — `pins.reorder` keeps what it leaves out in place."""
|
||||
pins.reorder(db, user, body.ids)
|
||||
db.commit()
|
||||
return {"ok": True}
|
||||
+253
-57
@@ -6,23 +6,29 @@ worker is a separate process); this module never downloads — it enqueues, mana
|
||||
serves finished files (range-aware, with the user's custom display name), and shares them.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app import audit, sysconfig
|
||||
from app.auth import admin_user, require_human
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.downloads import credentials as dlcreds
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads import urlguard
|
||||
from app.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
@@ -32,9 +38,11 @@ from app.models import (
|
||||
MediaAsset,
|
||||
User,
|
||||
)
|
||||
from app.routes._common import get_or_404, not_found, owned_or_404
|
||||
from app.security import hash_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api/downloads", tags=["downloads"])
|
||||
admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"])
|
||||
@@ -56,11 +64,22 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
as a raw URL for yt-dlp's generic extractor."""
|
||||
raw = (raw or "").strip()
|
||||
if not raw:
|
||||
raise HTTPException(status_code=400, detail="A video link or id is required.")
|
||||
raise refused(400, "link_required")
|
||||
if _VIDEO_ID.match(raw):
|
||||
return "youtube", raw
|
||||
if raw.startswith(("http://", "https://")):
|
||||
u = urlparse(raw)
|
||||
# `urlparse` raises ValueError on a malformed authority ("http://[::1/x"), and this runs
|
||||
# BEFORE the urlguard check that would otherwise turn it into a structured refusal — so
|
||||
# unguarded, that typo escapes as a 500. Answered in urlguard's own shape, because it is the
|
||||
# same typo class: ":99999" is caught there and localized, and one of the two arriving as a
|
||||
# bare English string would be exactly the split the canonical-patterns table forbids.
|
||||
try:
|
||||
u = urlparse(raw)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=422,
|
||||
detail={"code": "url_blocked", "reason": urlguard.REASON_MALFORMED},
|
||||
) from exc
|
||||
host = u.netloc.lower()
|
||||
if "youtube.com" in host:
|
||||
if u.path.startswith(("/shorts/", "/embed/", "/live/")):
|
||||
@@ -75,7 +94,7 @@ def resolve_source(raw: str) -> tuple[str, str]:
|
||||
if _VIDEO_ID.match(vid):
|
||||
return "youtube", vid
|
||||
return "url", raw
|
||||
raise HTTPException(status_code=400, detail="That doesn't look like a valid video link.")
|
||||
raise refused(400, "invalid_link")
|
||||
|
||||
|
||||
def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
@@ -89,6 +108,7 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"phase": job.phase,
|
||||
"error": job.error,
|
||||
"error_code": job.error_code,
|
||||
"error_detail": job.error_detail,
|
||||
"queue_pos": job.queue_pos,
|
||||
"created_at": iso(job.created_at),
|
||||
"source_kind": job.source_kind,
|
||||
@@ -122,6 +142,9 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"width": asset.width,
|
||||
"height": asset.height,
|
||||
"container": asset.container,
|
||||
# "video" | "audio" | "file" — the library gates play/edit on this (media only); a
|
||||
# generic uploaded file is download-only. Null while still in flight (set at finalize).
|
||||
"media_kind": asset.media_kind,
|
||||
"uploader_url": asset.uploader_url,
|
||||
"thumbnail_url": asset.thumbnail_url,
|
||||
"expires_at": iso(asset.expires_at),
|
||||
@@ -144,11 +167,30 @@ def _serialize_job(db: Session, job: DownloadJob) -> dict:
|
||||
return _serialize(job, asset)
|
||||
|
||||
|
||||
def _own_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None or job.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
return job
|
||||
def _own_job_for_update(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
"""The ownership check (`owned_or_404`) for a transition that moves its asset's ref count (resume, cancel, delete, redownload)
|
||||
or writes the status those decide on (pause). The job is re-read under a row lock, so two
|
||||
overlapping requests for one job run one after the other and the second decides on the first's
|
||||
committed result — never both releasing the same hold, and never a pause turning a job a cancel
|
||||
just released back into a holder; either can free a file another job still holds.
|
||||
|
||||
The asset is locked BEFORE the job: the order a finishing worker child takes them in
|
||||
(`service.finalize_if_running`), so the two can never deadlock."""
|
||||
while True:
|
||||
asset_id = owned_or_404(db, DownloadJob, job_id, user, what="download").asset_id
|
||||
if asset_id is not None:
|
||||
db.execute(select(MediaAsset.id).where(MediaAsset.id == asset_id).with_for_update())
|
||||
job = db.execute(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.id == job_id)
|
||||
.with_for_update()
|
||||
.execution_options(populate_existing=True)
|
||||
).scalar_one_or_none()
|
||||
if job is None:
|
||||
raise not_found("download")
|
||||
if job.asset_id == asset_id:
|
||||
return job
|
||||
db.rollback() # another request moved the job to another asset meanwhile: lock that one
|
||||
|
||||
|
||||
# --- profiles ------------------------------------------------------------------------------
|
||||
@@ -192,7 +234,7 @@ def create_profile(
|
||||
name = body.name.strip()
|
||||
spec = body.spec
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="A profile name is required.")
|
||||
raise refused(400, "profile_name_required")
|
||||
maxpos = db.execute(
|
||||
select(func.coalesce(func.max(DownloadProfile.sort_order), 100)).where(
|
||||
DownloadProfile.user_id == user.id
|
||||
@@ -220,16 +262,16 @@ def update_profile(
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
sent = body.model_fields_set
|
||||
if "name" in sent:
|
||||
name = (body.name or "").strip()
|
||||
if not name:
|
||||
raise HTTPException(status_code=400, detail="A profile name is required.")
|
||||
raise refused(400, "profile_name_required")
|
||||
p.name = name[:80]
|
||||
if "spec" in sent:
|
||||
if body.spec is None:
|
||||
raise HTTPException(status_code=400, detail="spec must be an object")
|
||||
raise refused(400, "spec_not_object")
|
||||
p.spec = body.spec
|
||||
db.commit()
|
||||
return _serialize_profile(p)
|
||||
@@ -241,12 +283,71 @@ def delete_profile(
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
db.delete(p)
|
||||
db.commit()
|
||||
return {"deleted": profile_id}
|
||||
|
||||
|
||||
# --- per-user download auth (cookies) ------------------------------------------------------
|
||||
# Write-only surface: the stored blob is NEVER returned — only the supported hosts + whether each
|
||||
# is configured + when. See downloads.credentials for the host-scope + cache-privacy guarantees.
|
||||
|
||||
|
||||
@router.get("/credentials")
|
||||
def list_credentials(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
return {"hosts": dlcreds.status_list(db, user.id)}
|
||||
|
||||
|
||||
class SetCredentialIn(BaseModel):
|
||||
cookies: str
|
||||
|
||||
|
||||
@router.put("/credentials/{host_key}")
|
||||
def set_credential(
|
||||
host_key: str,
|
||||
body: SetCredentialIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Store (encrypted) this user's cookies.txt for one supported host. Write-only: the value is
|
||||
never echoed back, logged, or audited — only the fact that it changed."""
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise not_found("host")
|
||||
try:
|
||||
cleaned = dlcreds.validate_cookies(body.cookies)
|
||||
except dlcreds.InvalidCookies as e:
|
||||
raise HTTPException(status_code=400, detail={"code": "cookies", "reason": str(e)})
|
||||
dlcreds.upsert_credential(db, user.id, host_key, cleaned)
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_CREDENTIAL_SET,
|
||||
target_type="download_credential",
|
||||
target_id=host_key,
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "hosts": dlcreds.status_list(db, user.id)}
|
||||
|
||||
|
||||
@router.delete("/credentials/{host_key}")
|
||||
def delete_credential(
|
||||
host_key: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise not_found("host")
|
||||
if dlcreds.delete_credential(db, user.id, host_key):
|
||||
audit.record(
|
||||
db,
|
||||
user.id,
|
||||
audit.AuditAction.DOWNLOAD_CREDENTIAL_DELETE,
|
||||
target_type="download_credential",
|
||||
target_id=host_key,
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "hosts": dlcreds.status_list(db, user.id)}
|
||||
|
||||
|
||||
def _resolve_spec(
|
||||
db: Session, user: User, profile_id: int | None, spec: dict | None
|
||||
) -> tuple[dict, int | None]:
|
||||
@@ -255,7 +356,7 @@ def _resolve_spec(
|
||||
if profile_id is not None:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or (not p.is_builtin and p.user_id != user.id):
|
||||
raise HTTPException(status_code=404, detail="Unknown profile")
|
||||
raise not_found("profile")
|
||||
return p.spec, p.id
|
||||
if spec is not None:
|
||||
return spec, None
|
||||
@@ -266,7 +367,7 @@ def _resolve_spec(
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if fallback is None:
|
||||
raise HTTPException(status_code=400, detail="No download profile available.")
|
||||
raise refused(400, "no_profile")
|
||||
return fallback.spec, fallback.id
|
||||
|
||||
|
||||
@@ -285,6 +386,17 @@ def enqueue_download(
|
||||
body: EnqueueDownloadIn, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
source_kind, source_ref = resolve_source(body.source)
|
||||
# SEC-12: a non-YouTube source is a raw URL the worker will fetch server-side, so refuse the
|
||||
# ones pointing back into our own network BEFORE a job exists. The worker re-checks (DNS can
|
||||
# move between here and the download); see downloads/urlguard.py for what that does and does
|
||||
# not cover.
|
||||
if source_kind == "url":
|
||||
try:
|
||||
urlguard.check_url(source_ref)
|
||||
except urlguard.BlockedUrl as e:
|
||||
raise HTTPException(
|
||||
status_code=422, detail={"code": "url_blocked", "reason": e.reason}
|
||||
) from e
|
||||
spec, profile_id = _resolve_spec(db, user, body.profile_id, body.spec)
|
||||
display_name = (body.display_name or "").strip() or None
|
||||
try:
|
||||
@@ -294,6 +406,61 @@ def enqueue_download(
|
||||
return _serialize_job(db, job)
|
||||
|
||||
|
||||
# Read the upload in 1 MiB chunks so a large file never sits fully in memory, and the size cap is
|
||||
# enforced on the ACTUAL bytes read (a client-set Content-Length can't be trusted to gate it).
|
||||
_UPLOAD_CHUNK = 1024 * 1024
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
def upload_download(
|
||||
file: UploadFile = File(...),
|
||||
display_name: str | None = Form(None),
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Ingest a local file into the caller's library (E7 S5). Any file type is accepted; media
|
||||
(video/audio) gets the full function set, a generic file is download-only. The file is streamed
|
||||
to a temp path under the download volume (enforcing the per-file cap as it goes), then handed to
|
||||
the non-yt-dlp ingest which probes + places it and returns an already-`done` job.
|
||||
|
||||
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||
plus sync DB writes — so it must NOT run on the event loop. `file.file` is the sync file object
|
||||
Starlette spools the multipart body into."""
|
||||
cap = sysconfig.get_int(db, "download_upload_max_bytes")
|
||||
staging = Path(settings.download_root) / ".staging" / "uploads"
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=str(staging))
|
||||
tmp_path = Path(tmp)
|
||||
try:
|
||||
size = 0
|
||||
with os.fdopen(fd, "wb") as out:
|
||||
while chunk := file.file.read(_UPLOAD_CHUNK):
|
||||
size += len(chunk)
|
||||
if cap and size > cap:
|
||||
raise HTTPException(
|
||||
status_code=422, detail={"code": "upload_too_large", "limit": cap}
|
||||
)
|
||||
out.write(chunk)
|
||||
if size == 0:
|
||||
raise HTTPException(status_code=400, detail={"code": "upload_empty"})
|
||||
try:
|
||||
job = uploadmod.ingest_upload(
|
||||
db,
|
||||
user.id,
|
||||
filename=file.filename or "upload",
|
||||
src_path=tmp_path,
|
||||
size=size,
|
||||
display_name=(display_name or "").strip() or None,
|
||||
)
|
||||
except quota.QuotaExceeded as e:
|
||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
||||
return _serialize_job(db, job)
|
||||
finally:
|
||||
# Ingest MOVES the temp file into place on success (unlink is then a no-op); on any early exit
|
||||
# — cap hit, empty, quota, probe/move error — this removes the abandoned staging file.
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class EnqueueEditIn(BaseModel):
|
||||
source_job_id: int = 0
|
||||
edit_spec: dict
|
||||
@@ -311,7 +478,7 @@ def enqueue_edit(
|
||||
source file is only read, never modified, so editing a shared video is safe."""
|
||||
source_job = _accessible_job(db, user, body.source_job_id)
|
||||
if source_job.status != "done" or source_job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="You can only edit a finished download.")
|
||||
raise refused(409, "edit_unfinished")
|
||||
spec = body.edit_spec
|
||||
display_name = (body.display_name or "").strip() or None
|
||||
try:
|
||||
@@ -410,7 +577,14 @@ def _clean_url(raw) -> str | None:
|
||||
return None
|
||||
if not url.startswith(("http://", "https://")):
|
||||
return None
|
||||
parsed = urlparse(url)
|
||||
# The third site in this module handing user input to `urlparse`, which raises on a malformed
|
||||
# authority. This one already has a "not a plausible URL" answer, so the typo belongs in it.
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
# silent-ok: a typo in a link someone pasted. None is this function's "not a plausible URL"
|
||||
# answer, the caller already has one, and any line here would carry the URL itself.
|
||||
except ValueError:
|
||||
return None
|
||||
if not parsed.netloc:
|
||||
return None
|
||||
return url[:2048]
|
||||
@@ -434,7 +608,7 @@ def update_download_meta(
|
||||
) -> dict:
|
||||
"""Update a download's display metadata: title, channel name + link, and extra reference URLs.
|
||||
All are display-only overrides — the on-disk file keeps its system-decided, id-bound name."""
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
sent = body.model_fields_set
|
||||
if "display_name" in sent:
|
||||
job.display_name = ((body.display_name or "").strip())[:255] or None
|
||||
@@ -454,7 +628,7 @@ def update_download_meta(
|
||||
def pause_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.pause(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -463,7 +637,7 @@ def pause_download(
|
||||
def resume_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.resume(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -478,16 +652,19 @@ def redownload_download(
|
||||
resolves to the newly-downloaded file — a re-download never breaks a link you already sent out
|
||||
(only a *failed* re-download leaves the job in `error`, which is the intended broken-link case).
|
||||
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the job
|
||||
id + current asset id) rather than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
The rebuild always re-fetches into a FRESH, private asset (the cache sig is salted with the new
|
||||
asset's own id, and the user's cookie salt when one applies — `credentials.rebuild_salt`) rather
|
||||
than the dedup pool. That means it (a) always genuinely rebuilds
|
||||
under the current format/naming rules — never silently reuses a shared cached file — and (b)
|
||||
never deletes a file another job/user still shares: the old asset is only removed if THIS job
|
||||
was its last holder."""
|
||||
job = _own_job(db, user, job_id)
|
||||
if job.job_kind == "edit":
|
||||
raise HTTPException(status_code=409, detail="Editor clips can't be re-downloaded.")
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
if job.job_kind in ("edit", "upload"):
|
||||
# Neither has a re-fetchable source: an editor clip is derived locally, an upload IS the
|
||||
# source. Re-running yt-dlp against source_kind='upload'/'edit' would just fail.
|
||||
raise refused(409, "not_redownloadable")
|
||||
if job.status in ("queued", "running", "paused"):
|
||||
raise HTTPException(status_code=409, detail="This download is still in progress.")
|
||||
raise refused(409, "still_downloading")
|
||||
service.redownload(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -496,7 +673,7 @@ def redownload_download(
|
||||
def cancel_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.cancel(db, job)
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -505,7 +682,7 @@ def cancel_download(
|
||||
def delete_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.delete(db, job)
|
||||
return {"deleted": job_id}
|
||||
|
||||
@@ -516,7 +693,7 @@ def delete_download(
|
||||
def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
raise not_found("download")
|
||||
if job.user_id == user.id:
|
||||
return job
|
||||
shared = db.execute(
|
||||
@@ -525,7 +702,7 @@ def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
)
|
||||
).first()
|
||||
if shared is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown download")
|
||||
raise not_found("download")
|
||||
return job
|
||||
|
||||
|
||||
@@ -539,10 +716,10 @@ def download_file(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=409, detail="This download isn't ready.")
|
||||
raise refused(409, "download_not_ready")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.rel_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="File is no longer available.")
|
||||
raise not_found("file")
|
||||
|
||||
filename = storage.download_filename(
|
||||
job.display_name or asset.title or asset.source_ref, asset.container, path
|
||||
@@ -550,8 +727,21 @@ def download_file(
|
||||
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
# Starlette's FileResponse honours the Range header (206 partial content) for seek/resume.
|
||||
return FileResponse(path, filename=filename)
|
||||
# The served Content-Type comes from the ffprobe-verified media_kind + a container allow-list
|
||||
# (storage.served_media_type), NEVER guessed from the file's own extension. A generic 'file' (and
|
||||
# any media whose container we don't recognise — e.g. a media/HTML polyglot named "evil.html")
|
||||
# returns None → a neutral force-download attachment, so uploaded HTML/SVG can't execute inline on
|
||||
# this origin. Recognised media is served with its real type so the in-app player streams it. Both
|
||||
# are range-aware (FileResponse → 206).
|
||||
media_type = storage.served_media_type(asset.media_kind, asset.container)
|
||||
if media_type is None:
|
||||
return FileResponse(
|
||||
path,
|
||||
filename=filename,
|
||||
media_type="application/octet-stream",
|
||||
content_disposition_type="attachment",
|
||||
)
|
||||
return FileResponse(path, filename=filename, media_type=media_type)
|
||||
|
||||
|
||||
# --- editor filmstrip (scrub track) --------------------------------------------------------
|
||||
@@ -568,7 +758,7 @@ def storyboard_meta(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise HTTPException(status_code=409, detail="This download isn't ready.")
|
||||
raise refused(409, "download_not_ready")
|
||||
meta = editmod.ensure_storyboard(asset, settings.download_root)
|
||||
if meta is None:
|
||||
return {"available": False}
|
||||
@@ -589,10 +779,10 @@ def poster_image(job_id: int, user: User = Depends(require_human), db: Session =
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.poster_path:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.poster_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No poster.")
|
||||
raise not_found("poster")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
|
||||
@@ -603,10 +793,10 @@ def storyboard_image(
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
if asset is None or not asset.storyboard_path:
|
||||
raise HTTPException(status_code=404, detail="No filmstrip.")
|
||||
raise not_found("filmstrip")
|
||||
path = storage.safe_abs_path(settings.download_root, asset.storyboard_path)
|
||||
if path is None:
|
||||
raise HTTPException(status_code=404, detail="No filmstrip.")
|
||||
raise not_found("filmstrip")
|
||||
return FileResponse(path, media_type="image/jpeg")
|
||||
|
||||
|
||||
@@ -624,21 +814,21 @@ def share_download(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="Only a finished download can be shared.")
|
||||
raise refused(409, "share_unfinished")
|
||||
target = body.email.strip().lower()
|
||||
if not target:
|
||||
raise HTTPException(status_code=400, detail="A recipient email is required.")
|
||||
raise refused(400, "recipient_required")
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == target)
|
||||
).scalar_one_or_none()
|
||||
# Uniform "no user" for a missing OR non-messageable (demo/suspended/deactivated) target, so
|
||||
# sharing can't be used to probe which addresses exist or to reach a suspended account.
|
||||
if not is_messageable_user(recipient):
|
||||
raise HTTPException(status_code=404, detail="No user with that email.")
|
||||
raise not_found("recipient")
|
||||
if recipient.id == user.id:
|
||||
raise HTTPException(status_code=400, detail="That's already your download.")
|
||||
raise refused(400, "share_own")
|
||||
exists = db.execute(
|
||||
select(DownloadShare.id).where(
|
||||
DownloadShare.job_id == job_id, DownloadShare.to_user_id == recipient.id
|
||||
@@ -657,7 +847,7 @@ def share_download(
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
return {"shared_with": recipient.email}
|
||||
return {"ok": True, "shared_with": recipient.email}
|
||||
|
||||
|
||||
@router.delete("/{job_id}/share/{email}")
|
||||
@@ -667,7 +857,9 @@ def unshare_download(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
_own_job(db, user, job_id) # ownership guard (404s if not the caller's job)
|
||||
owned_or_404(
|
||||
db, DownloadJob, job_id, user, what="download"
|
||||
) # ownership guard (404s if not the caller's job)
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == email.strip().lower())
|
||||
).scalar_one_or_none()
|
||||
@@ -699,7 +891,7 @@ def remove_shared_with_me(
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"removed": job_id}
|
||||
return {"ok": True, "removed": job_id}
|
||||
|
||||
|
||||
# --- recipients (for the internal "share with a user" picker) ------------------------------
|
||||
@@ -729,10 +921,10 @@ def share_recipients(
|
||||
def _own_link(db: Session, user: User, link_id: int) -> DownloadLink:
|
||||
link = db.get(DownloadLink, link_id)
|
||||
if link is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown link")
|
||||
raise not_found("link")
|
||||
job = db.get(DownloadJob, link.job_id)
|
||||
if job is None or job.user_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="Unknown link")
|
||||
raise not_found("link")
|
||||
return link
|
||||
|
||||
|
||||
@@ -748,7 +940,7 @@ def _link_expiry(days: int | None) -> datetime | None:
|
||||
def list_links(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
_own_job(db, user, job_id)
|
||||
owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
rows = (
|
||||
db.execute(
|
||||
select(DownloadLink)
|
||||
@@ -763,7 +955,12 @@ def list_links(
|
||||
|
||||
class CreateLinkIn(BaseModel):
|
||||
password: str = ""
|
||||
allow_download: bool = False
|
||||
# D1 (E7 S3): a newly shared download is downloadable by default — the common intent is "let the
|
||||
# recipient have the file", not stream-only. Forward-only: this only sets the default for links
|
||||
# minted from now on; existing links keep whatever `allow_download` they were created with (the
|
||||
# column default stays False, so no back-catalogue link silently flips). The owner can still
|
||||
# uncheck it in the Share dialog, and toggle any link stream-only afterwards.
|
||||
allow_download: bool = True
|
||||
expires_days: int | None = None
|
||||
|
||||
|
||||
@@ -774,9 +971,9 @@ def create_link(
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
raise HTTPException(status_code=409, detail="Only a finished download can be shared.")
|
||||
raise refused(409, "share_unfinished")
|
||||
pw = body.password.strip()
|
||||
link = DownloadLink(
|
||||
job_id=job_id,
|
||||
@@ -921,8 +1118,7 @@ def admin_set_quota(
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if db.get(User, user_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown user")
|
||||
get_or_404(db, User, user_id, what="user")
|
||||
cur = quota.resolve(db, user_id)
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is None:
|
||||
|
||||
+45
-18
@@ -1,7 +1,7 @@
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -14,6 +14,7 @@ from app.feedquery import (
|
||||
decode_cursor,
|
||||
encode_cursor,
|
||||
feed_summary,
|
||||
feed_years,
|
||||
filtered_query,
|
||||
keyset_predicate,
|
||||
order_by,
|
||||
@@ -21,15 +22,18 @@ from app.feedquery import (
|
||||
sort_keys,
|
||||
)
|
||||
from app.models import (
|
||||
POSITION_MAX_S,
|
||||
ChannelTag,
|
||||
Tag,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.routes._common import get_or_404, youtube_refusal, not_found
|
||||
from app.sync.videos import parse_iso8601_duration
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["feed"])
|
||||
|
||||
@@ -53,6 +57,8 @@ def _feed_params(
|
||||
tag_mode_language: str = "or",
|
||||
tag_mode_topic: str = "or",
|
||||
tag_mode_other: str = "or",
|
||||
# "No tag" filter: keep only channels the user has not personally tagged (no user-owned ChannelTag).
|
||||
untagged: bool = False,
|
||||
channel_ids: list[str] = Query(default=[]),
|
||||
q: str | None = None,
|
||||
min_duration: int | None = None,
|
||||
@@ -74,6 +80,7 @@ def _feed_params(
|
||||
"topic": tag_mode_topic,
|
||||
"other": tag_mode_other,
|
||||
},
|
||||
"untagged": untagged,
|
||||
"channel_ids": channel_ids,
|
||||
"q": q,
|
||||
"min_duration": min_duration,
|
||||
@@ -114,7 +121,7 @@ def get_feed(
|
||||
vals, cursor_id = decode_cursor(keys, cursor)
|
||||
except ValueError:
|
||||
# `from None`: the ValueError is an expected bad-input signal, not an error to chain.
|
||||
raise HTTPException(status_code=400, detail="Invalid feed cursor") from None
|
||||
raise refused(400, "invalid_cursor") from None
|
||||
for k, v in zip(keys, vals):
|
||||
k["value"] = v
|
||||
query = query.where(keyset_predicate(keys, cursor_id))
|
||||
@@ -203,6 +210,19 @@ def get_feed_summary(
|
||||
return feed_summary(db, user, **params)
|
||||
|
||||
|
||||
@router.get("/feed/years")
|
||||
def get_feed_years(
|
||||
params: dict = Depends(_feed_params),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Per-year video counts for the year control (item 8 / E8·S2): the quick recent-year chips and the
|
||||
"more…" per-year-counts dialog. Every filter EXCEPT the publish-date narrowing applies, so picking a
|
||||
year re-queries the feed but not this list. Response: {"years": [{"year": <int>, "count": <int>}]},
|
||||
newest year first."""
|
||||
return {"years": feed_years(db, user, **params)}
|
||||
|
||||
|
||||
class VideoStateIn(BaseModel):
|
||||
status: str
|
||||
|
||||
@@ -216,9 +236,8 @@ def set_video_state(
|
||||
) -> dict:
|
||||
status = body.status
|
||||
if status not in VALID_STATES:
|
||||
raise HTTPException(status_code=400, detail=f"status must be one of {VALID_STATES}")
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise refused(400, "invalid_status")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
@@ -238,9 +257,11 @@ def set_video_state(
|
||||
db.delete(row)
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: the documented lost race — a concurrent player checkpoint already removed
|
||||
# or moved this row. The state the caller asked for is the state that holds either way.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "status": "new"}
|
||||
return {"ok": True, "video_id": video_id, "status": "new"}
|
||||
|
||||
# watched / hidden: atomic upsert. The player marks "watched" at end-of-playback at
|
||||
# the same instant a progress checkpoint may DELETE the in-progress row; a plain
|
||||
@@ -261,7 +282,7 @@ def set_video_state(
|
||||
)
|
||||
db.execute(stmt)
|
||||
db.commit()
|
||||
return {"video_id": video_id, "status": status}
|
||||
return {"ok": True, "video_id": video_id, "status": status}
|
||||
|
||||
|
||||
@router.delete("/videos/{video_id}/state")
|
||||
@@ -280,17 +301,23 @@ def clear_video_state(
|
||||
db.delete(row)
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
|
||||
|
||||
class VideoProgressIn(BaseModel):
|
||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||
# one into a 500.
|
||||
model_config = ConfigDict(allow_inf_nan=False)
|
||||
|
||||
# Accept a float and truncate (below) — the player may post a raw currentTime (a float), which
|
||||
# the old `int(payload.get(...))` silently truncated. A plain `int` field would 422 that on the
|
||||
# 10s progress hot path, losing the resume position; `float` keeps the old tolerance.
|
||||
position_seconds: float = 0
|
||||
duration_seconds: float = 0
|
||||
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
||||
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
|
||||
|
||||
@router.post("/videos/{video_id}/progress")
|
||||
@@ -309,8 +336,7 @@ def set_video_progress(
|
||||
if position < 0:
|
||||
position = 0
|
||||
|
||||
if db.get(Video, video_id) is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
# Decide whether this position is worth keeping (else clear it).
|
||||
near_end = duration > 0 and position > duration - FINISH_MARGIN_SECONDS
|
||||
@@ -333,9 +359,10 @@ def set_video_progress(
|
||||
# have removed/updated this row already — tolerate the lost race.
|
||||
try:
|
||||
db.commit()
|
||||
# silent-ok: exactly the race the comment above describes; the response stays true.
|
||||
except StaleDataError:
|
||||
db.rollback()
|
||||
return {"video_id": video_id, "position_seconds": 0}
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": 0}
|
||||
|
||||
if row is None:
|
||||
row = VideoState(user_id=user.id, video_id=video_id)
|
||||
@@ -343,7 +370,7 @@ def set_video_progress(
|
||||
row.position_seconds = position
|
||||
row.progress_updated_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
return {"video_id": video_id, "position_seconds": position}
|
||||
return {"ok": True, "video_id": video_id, "position_seconds": position}
|
||||
|
||||
|
||||
@router.get("/videos/{video_id}")
|
||||
@@ -376,7 +403,7 @@ def get_video_detail(
|
||||
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit, so the
|
||||
# demo account can't reach it — it just sees "unknown" for a video we don't already store.
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise not_found("video")
|
||||
|
||||
try:
|
||||
with (
|
||||
@@ -385,9 +412,9 @@ def get_video_detail(
|
||||
):
|
||||
items = yt.get_videos([video_id])
|
||||
except YouTubeError as exc:
|
||||
raise HTTPException(status_code=422, detail=f"YouTube lookup failed: {exc}")
|
||||
raise youtube_refusal(exc)
|
||||
if not items:
|
||||
raise HTTPException(status_code=404, detail="Unknown video")
|
||||
raise not_found("video")
|
||||
|
||||
snippet = items[0].get("snippet", {})
|
||||
stats = items[0].get("statistics", {})
|
||||
|
||||
+102
-25
@@ -1,11 +1,15 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, RootModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import (
|
||||
count_admins,
|
||||
count_pending_activations,
|
||||
current_user,
|
||||
drop_from_wallet,
|
||||
google_enabled,
|
||||
has_read_scope,
|
||||
has_write_scope,
|
||||
@@ -13,9 +17,16 @@ from app.auth import (
|
||||
optional_current_user,
|
||||
purge_user,
|
||||
)
|
||||
from app import debugsession, diagnostics
|
||||
from app.issues import gitea
|
||||
from app import email as email_mod
|
||||
from app import logs
|
||||
from app import sysconfig
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/me", tags=["me"])
|
||||
|
||||
@@ -62,28 +73,28 @@ def switch_account(
|
||||
here) are allowed, and access is re-checked in case the invite was revoked since."""
|
||||
target = body.user_id
|
||||
if target not in (request.session.get("account_ids") or []):
|
||||
raise HTTPException(status_code=403, detail="Not an account you've signed into here.")
|
||||
raise refused(403, "not_your_account")
|
||||
# An account `_session_user` would refuse — gone, deactivated, suspended, or its session revoked — is
|
||||
# refused here too, and dropped from the switch list the way `_session_user` drops it. Made the session
|
||||
# default instead, the next request's current_user would refuse it as the DEFAULT and clear the whole
|
||||
# wallet session, logging out every account here. (B-core-9)
|
||||
u = db.get(User, target)
|
||||
if u is None:
|
||||
raise HTTPException(status_code=404, detail="That account no longer exists.")
|
||||
drop_from_wallet(request.session, target)
|
||||
raise not_found("account")
|
||||
if not is_allowed(db, u.email):
|
||||
raise HTTPException(status_code=403, detail="That account no longer has access.")
|
||||
raise refused(403, "account_no_access")
|
||||
if not u.is_active:
|
||||
drop_from_wallet(request.session, target)
|
||||
raise refused(403, "account_no_access")
|
||||
if u.is_suspended:
|
||||
# Don't make a suspended account the active one — the next request's current_user would
|
||||
# see the suspension and clear the WHOLE wallet session, logging out every account here.
|
||||
raise HTTPException(status_code=403, detail="That account is suspended.")
|
||||
# Same reasoning for a REVOKED session (a password reset/change or "log out everywhere" bumped the
|
||||
# account's epoch): making it the session default would trip current_user's epoch check on the next
|
||||
# request and clear the whole wallet. Mirror the suspend guard — drop the stale account from the
|
||||
# switch list and refuse, rather than logging everyone out. (B-core-9)
|
||||
drop_from_wallet(request.session, target)
|
||||
raise refused(403, "account_suspended")
|
||||
# Revoked: a password reset/change or "log out everywhere" bumped the account's epoch.
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(target), 0)) != (u.session_epoch or 0):
|
||||
request.session["account_ids"] = [
|
||||
a for a in (request.session.get("account_ids") or []) if a != target
|
||||
]
|
||||
raise HTTPException(
|
||||
status_code=403, detail="That account was signed out elsewhere — sign in again."
|
||||
)
|
||||
drop_from_wallet(request.session, target)
|
||||
raise refused(403, "account_signed_out")
|
||||
request.session["user_id"] = target
|
||||
return {"ok": True, "user_id": target}
|
||||
|
||||
@@ -98,11 +109,21 @@ def get_me(
|
||||
if user is None:
|
||||
return {"authenticated": False}
|
||||
pending_invites = 0
|
||||
pending_activations = 0
|
||||
# Admin-only, because it drives an operator warning: email is fail-soft, so a half-filled SMTP
|
||||
# config costs verification links, password resets and access-request notices with nothing
|
||||
# visible anywhere. Surfacing it here puts the warning on the screens where onboarding happens.
|
||||
email_configured = True
|
||||
if user.role == "admin":
|
||||
pending_invites = (
|
||||
db.scalar(select(func.count()).select_from(Invite).where(Invite.status == "pending"))
|
||||
or 0
|
||||
)
|
||||
# Counted separately from pending_invites rather than folded into it: the two are different
|
||||
# facts (a request awaiting a decision vs a decided address whose sign-up never got let in),
|
||||
# and quietly widening one number would make an existing field mean something new.
|
||||
pending_activations = count_pending_activations(db)
|
||||
email_configured = email_mod.email_enabled(db) # this request's session; no second connect
|
||||
return {
|
||||
"authenticated": True,
|
||||
"id": user.id,
|
||||
@@ -112,7 +133,16 @@ def get_me(
|
||||
"role": user.role,
|
||||
"is_demo": user.is_demo,
|
||||
"has_google": user.google_sub is not None,
|
||||
# The address Google answers with, when it DIFFERS from the account's own. Same string for
|
||||
# most people, and then this is None so no surface has to decide whether to show it — the
|
||||
# comparison lives here, once, instead of in every consumer.
|
||||
"google_email": (
|
||||
user.google_email if user.google_email and user.google_email != user.email else None
|
||||
),
|
||||
"has_password": user.password_hash is not None,
|
||||
# The account's latest sign-in on any device — shown in Settings → Account, so a sign-in the
|
||||
# user does not recognise is something they can see (null: none since 0.88, or the demo).
|
||||
"last_login_at": iso(user.last_login_at),
|
||||
# Instance-wide: whether Google sign-in / YouTube connect is available at all (the UI hides
|
||||
# YouTube-access affordances and the onboarding nudge when it isn't configured).
|
||||
"google_enabled": google_enabled(db),
|
||||
@@ -121,10 +151,26 @@ def get_me(
|
||||
"can_read": has_read_scope(user),
|
||||
"can_write": has_write_scope(user),
|
||||
"pending_invites": pending_invites,
|
||||
# Sign-ups an admin still has to let in (roadmap UX-25): approved BEFORE they registered, so
|
||||
# nothing ever reached the Access-requests queue and the badge used to read zero.
|
||||
"pending_activations": pending_activations,
|
||||
"email_configured": email_configured,
|
||||
"preferences": user.preferences or {},
|
||||
# The consented diagnostics level in force and when it lapses (E22 S5). The client applies
|
||||
# it to its own log and streams that log while it is raised.
|
||||
"diagnostics": diagnostics.state(db, user),
|
||||
# The live debug session, if one is recording (E16) — a reload resumes it.
|
||||
"debug_session": _live_session(db, user),
|
||||
# Where a bug report goes (E16 S4): the tracker's repository, or None — download only.
|
||||
"issues": gitea.target(db),
|
||||
}
|
||||
|
||||
|
||||
def _live_session(db: Session, user: User) -> dict | None:
|
||||
session = debugsession.open_session(db, user)
|
||||
return None if session is None else debugsession.view(db, session)
|
||||
|
||||
|
||||
@router.delete("/account")
|
||||
def delete_my_account(
|
||||
request: Request,
|
||||
@@ -137,13 +183,15 @@ def delete_my_account(
|
||||
which all cascade on the users row. Quota-audit events are kept but anonymised (FK SET NULL).
|
||||
The shared demo account can't be deleted, and the last remaining admin can't delete itself
|
||||
(that would lock everyone out of the admin surfaces)."""
|
||||
# Structured, like the other three demo refusals: `req` leaves 403 to the caller and
|
||||
# `notifyActionError` discards a plain-string detail in favour of the caller's translated
|
||||
# fallback, so a hand-written English sentence here reaches a Hungarian UI as a generic
|
||||
# "couldn't do that" — or, on this affordance, as nothing at all. `reason` distinguishes it
|
||||
# from the other sites' bare `{code: "demo"}` so the UI can say what specifically is refused.
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail="The shared demo account can't be deleted.")
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "delete_account"})
|
||||
if user.role == "admin" and count_admins(db) <= 1:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="You're the only admin — promote another admin before deleting your account.",
|
||||
)
|
||||
raise refused(400, "only_admin_delete")
|
||||
user_id = user.id
|
||||
# Full erasure (cascades) + access-request cleanup + Google-grant revocation; shared with the
|
||||
# admin delete path. Capture the id first — `user` is detached once purge_user deletes the row.
|
||||
@@ -158,16 +206,45 @@ def delete_my_account(
|
||||
return {"deleted": True}
|
||||
|
||||
|
||||
class PreferencesIn(RootModel[dict[str, Any]]):
|
||||
"""Free-form UI preferences — a JSON object of any keys (theme, filters, …), merged into the
|
||||
stored ones. A model rather than a bare `dict` parameter so a non-object body is a 422 at the
|
||||
boundary like every other request body."""
|
||||
|
||||
|
||||
@router.put("/preferences")
|
||||
def update_preferences(
|
||||
preferences: dict,
|
||||
preferences: PreferencesIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# Merge so partial updates don't wipe other keys.
|
||||
merged = dict(user.preferences or {})
|
||||
merged.update(preferences)
|
||||
merged.update(preferences.root)
|
||||
user.preferences = merged
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return {"preferences": merged}
|
||||
return {"ok": True, "preferences": merged}
|
||||
|
||||
|
||||
class DiagnosticsIn(BaseModel):
|
||||
level: diagnostics.DiagnosticsLevel
|
||||
|
||||
|
||||
@router.put("/diagnostics")
|
||||
def update_diagnostics(
|
||||
body: DiagnosticsIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Raise the diagnostics level (the user consented in Settings → Diagnosztika) or return to the
|
||||
base. The expiry is the server's: a raised level always lapses after
|
||||
`diagnostics.ELEVATION_HOURS`, whatever the client asks. The shared demo account cannot raise
|
||||
it — its log would interleave every visitor's session under one account."""
|
||||
if user.is_demo and body.level != diagnostics.BASE:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "diagnostics"})
|
||||
diagnostics.set_level(user, body.level)
|
||||
db.commit()
|
||||
# The rest of THIS request, and the process's elevation registry, follow the new level now.
|
||||
logs.set_user(user.id, diagnostics.effective_level(user), user.diagnostics_until)
|
||||
return {"ok": True, "diagnostics": diagnostics.state(db, user)}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user