Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
20178f0ec4 | ||
|
|
3865689c26 | ||
|
|
c32429ba69 | ||
|
|
a504550196 | ||
|
|
5095047c81 | ||
|
|
4e70f168b1 | ||
|
|
7f17f488ab | ||
|
|
d1704a5519 | ||
|
|
121ebeebc1 | ||
|
|
ee5290ba01 | ||
|
|
dd6a666ad1 | ||
|
|
392099770a | ||
|
|
2aa3ac6e85 | ||
|
|
891dc628f3 | ||
|
|
0a35f2a3b1 | ||
|
|
d25a132495 | ||
|
|
61d3a1a189 | ||
|
|
d482a5e8e2 | ||
|
|
2064373ac2 | ||
|
|
0cda5b6dd4 | ||
|
|
f3c94641a7 | ||
|
|
59b27467ab | ||
|
|
02feaa9a1c | ||
|
|
c1058f1d5a | ||
|
|
23f16b69df | ||
|
|
c813814c3c | ||
|
|
ed6facb703 | ||
|
|
d44d8d6dda | ||
|
|
b20e9a3947 | ||
|
|
6073a006b8 | ||
|
|
dba4758c59 | ||
|
|
68cfc7e25d | ||
|
|
00886d92fe | ||
|
|
e7d2eb8cf6 | ||
|
|
e886890d6e | ||
|
|
5561d9a379 | ||
|
|
2768ab5bdd | ||
|
|
8c79590165 | ||
|
|
eef3ac9c5c | ||
|
|
4ba7c966d7 | ||
|
|
513dd8175a | ||
|
|
c4298b0a73 | ||
|
|
5ca2a0e6ee | ||
|
|
5257d57196 | ||
|
|
58ad95263c | ||
|
|
96ad1ef306 | ||
|
|
d1bbe4524c | ||
|
|
eecc195b7e | ||
|
|
2f901ad643 | ||
|
|
1f357b0c21 | ||
|
|
fb962682eb | ||
|
|
4e71723a07 | ||
|
|
4aaaaa7860 | ||
|
|
c73d8be094 | ||
|
|
30545d0d88 | ||
|
|
ac702192c9 | ||
|
|
24c47d12bd | ||
|
|
ab76166878 | ||
|
|
a42f346a32 | ||
|
|
f2fe0fc0df | ||
|
|
ca707fd390 | ||
|
|
2eb20ca227 | ||
|
|
adaa7dc23c | ||
|
|
91f6a77fa3 | ||
|
|
36653886d5 | ||
|
|
7e681e112e | ||
|
|
3642397f63 | ||
|
|
6d8b7be6d8 | ||
|
|
cca3315836 | ||
|
|
7f08c510bf | ||
|
|
a499f00209 | ||
|
|
c2bf8fb7f6 | ||
|
|
11d35910fd | ||
|
|
27f48b7a8b | ||
|
|
bc856ab2af | ||
|
|
0dfb4a984b | ||
|
|
5bcd1d8d8a | ||
|
|
f49789665e | ||
|
|
0e077b84c8 | ||
|
|
ce8ccfc3c0 | ||
|
|
33dec56e45 | ||
|
|
2a699e26ba | ||
|
|
13c8c05661 | ||
|
|
cc414de952 |
+1
-1
@@ -28,7 +28,7 @@ ALLOWED_EMAILS=
|
||||
# Admin emails (subset of the above) get the admin role.
|
||||
ADMIN_EMAILS=
|
||||
|
||||
# Logging (E22). LOG_LEVEL is the BOOTSTRAP level — the level a process starts at and the only one
|
||||
# Logging. LOG_LEVEL is the BOOTSTRAP level — the level a process starts at and the only one
|
||||
# the worker/download child read; the API's running level is the admin Configuration page's
|
||||
# `log_level` knob, which takes effect without a restart. TRACE | DEBUG | INFO | WARNING | ERROR, in
|
||||
# any letter case; any other value stops the boot with an error. LOG_FORMAT is "json" (one object per
|
||||
|
||||
@@ -2,8 +2,11 @@
|
||||
# whole-repo reformatting does not bury the real authorship of each line.
|
||||
# Configure once per clone: git config blame.ignoreRevsFile .git-blame-ignore-revs
|
||||
#
|
||||
# Prettier one-time repo-wide format pass (R2 S1b, 2026-07-21)
|
||||
# Prettier one-time repo-wide format pass (2026-07-21)
|
||||
51c36c2071728d2cd7ae580b7bfdf21f6867a347
|
||||
|
||||
# ruff format one-time backend + scripts pass (line-length 100, 2026-08-23)
|
||||
b1832afd180866b922521148603db447020c57a4
|
||||
|
||||
# Plan-code purge from comments, one-time (2026-10-04)
|
||||
4aaaaa786017278460e0b4f94cc0d3374e0e37e2
|
||||
|
||||
@@ -1,128 +0,0 @@
|
||||
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
||||
# independent of any one machine. Two jobs match the two local lanes:
|
||||
# frontend — tsc (3 projects) + eslint + prettier --check + knip (both modes) + vitest + vite build
|
||||
# + npm audit of the production dependencies
|
||||
# backend — ruff + basedpyright + the full pytest suite (pure-logic + the DB-backed lane) against
|
||||
# a real Postgres + pip-audit of the lock
|
||||
#
|
||||
# They are declared side by side but they DO NOT run in parallel: the lab has a single act_runner
|
||||
# with one slot, so they queue. Measured across runs #86-#91 — the backend job starts at the exact
|
||||
# second the frontend one ends, every time, and the pair costs ~4 + ~6 = **~10 minutes from the
|
||||
# push**. Anything that waits on this gate has to budget for the sum, not the max; the ops
|
||||
# `release` read-back timed out on its first real run for believing otherwise.
|
||||
#
|
||||
# Runner: the lab act_runner (LXC 115) mounts the Docker socket, so service containers work. The backend
|
||||
# runs pytest NATIVELY against a `postgres` service (not the local docker-compose.test.yml, whose
|
||||
# bind-mount is unreliable under a mounted-socket runner); fidelity is kept by installing the SAME pinned
|
||||
# requirements, Postgres 16, and ffmpeg, and by setting SIFTLODE_TEST_REQUIRE_DB=1 so a DB/ffmpeg that
|
||||
# failed to come up is a hard error, never a silent skip.
|
||||
#
|
||||
# The frontend container's node major is NOT free: it must equal the prod build image's and the dev
|
||||
# machine's, because vitest/tsc read the engine's own Intl and web-API implementations and those split
|
||||
# between majors. `.nvmrc` is the single source of truth and
|
||||
# `frontend/src/lib/nodeEngine.contract.test.ts` fails this gate if any of them drift apart.
|
||||
name: gate
|
||||
|
||||
# Branch pushes only. A release pushes `main` and then the `v<ver>` tag, and a bare `on: [push]`
|
||||
# treated the tag as a second push of the IDENTICAL commit — every release ran this whole gate
|
||||
# twice, to the same verdict, for nothing (runs #81-#90 are five such pairs). `branches: ['**']`
|
||||
# keeps every branch covered and drops tag refs.
|
||||
#
|
||||
# It has to be `branches`, NOT the `tags-ignore` this obviously wants to be. Gitea's matchPushEvent
|
||||
# requires EVERY named filter to match, and each one bails on the wrong ref kind: `tags-ignore`
|
||||
# starts with `if !refName.IsTag() { break }`, so a filter meant to skip tags rejects every BRANCH
|
||||
# push and the gate stops running entirely. `branches` bails the same way on a tag, which is the
|
||||
# direction we want. (Measured in go-gitea v1.27.3 modules/actions/workflows.go, not assumed — it
|
||||
# differs from GitHub, where `tags-ignore` alone leaves branches unfiltered.)
|
||||
#
|
||||
# If that filter were ever wrong the gate would not run at all — which is why `siftlode release`
|
||||
# now ends by reading the verdict back for the sha it pushed, and says so out loud when there is
|
||||
# no run for it. A gate that silently stops running is the failure this repo already lived through.
|
||||
on:
|
||||
push:
|
||||
branches: ["**"]
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
container: node:24-bookworm
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: npm ci
|
||||
- run: npm run typecheck
|
||||
- run: npm run lint
|
||||
- run: npm run format:check
|
||||
- run: npm run knip
|
||||
- run: npm run test
|
||||
# The production bundle (E32): before this, `vite build` — and devOnlyGuard, which fails it if
|
||||
# dev-only code reaches the bundle — first ran inside the image build, after main was pushed.
|
||||
- run: npx vite build
|
||||
# What ships to the browser: a known CVE at high or above in a production dependency fails.
|
||||
# (The dev tooling's own advisories are burned down by hand; see docs/gates.md.)
|
||||
- run: npm audit --omit=dev --audit-level=high
|
||||
|
||||
backend:
|
||||
runs-on: ubuntu-latest
|
||||
# A combined python+node image: python 3.12 (matches the prod Dockerfile) for the tests, and node
|
||||
# so the JS actions (checkout) run. actions/setup-python is NOT usable here — the runner can't reach
|
||||
# GitHub's python-versions download, so it must be baked into the image.
|
||||
#
|
||||
# This node runs the checkout action, never our code, so it is deliberately OUTSIDE the engine
|
||||
# contract the frontend job is held to — it tracks `.nvmrc` as hygiene (node 20 went EOL in April
|
||||
# 2026), but a missing upstream tag here must never be able to block the gate.
|
||||
container: nikolaik/python-nodejs:python3.12-nodejs24
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: siftlode
|
||||
POSTGRES_PASSWORD: siftlode
|
||||
POSTGRES_DB: siftlode_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U siftlode -d siftlode_test"
|
||||
--health-interval 2s
|
||||
--health-timeout 3s
|
||||
--health-retries 30
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg://siftlode:siftlode@postgres:5432/siftlode_test
|
||||
SIFTLODE_TEST_REQUIRE_DB: "1"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# ffmpeg for the poster/thumbnail tests; postgresql-client for the readiness wait below.
|
||||
- run: apt-get update && apt-get install -y --no-install-recommends ffmpeg postgresql-client
|
||||
- name: Wait for Postgres
|
||||
run: until pg_isready -h postgres -U siftlode -d siftlode_test; do sleep 1; done
|
||||
- run: pip install --no-cache-dir -r backend/requirements.lock -r backend/requirements-dev.txt
|
||||
- name: ruff
|
||||
run: python -m ruff check --config backend/ruff.toml backend scripts
|
||||
- name: ruff format --check
|
||||
run: python -m ruff format --check --config backend/ruff.toml backend scripts
|
||||
# The ninth lane (E27 S3). Run from backend/ so it resolves exactly as the local lane does —
|
||||
# pyrightconfig.json's include/extraPaths are relative to the config, but the baseline is
|
||||
# found relative to the working directory. New errors fail; the ~200 pre-existing ones are in
|
||||
# .basedpyright/baseline.json and burn down epic by epic. Matching is per (file, rule, column
|
||||
# span) plus a count, so a new error of the SAME shape in a file where a sibling was just
|
||||
# fixed is absorbed — burn the hot files down first.
|
||||
#
|
||||
# The second step is not redundant. basedpyright TIGHTENS the baseline in place when the count
|
||||
# drops, with no --writebaseline, and CI throws its workspace away — so without this a
|
||||
# burn-down is green here and red on the developer's `siftlode check`, which has the same
|
||||
# guard. Two gates over one tree must not answer differently.
|
||||
- name: basedpyright
|
||||
run: python -m basedpyright --project pyrightconfig.json
|
||||
working-directory: backend
|
||||
- name: basedpyright baseline is committed
|
||||
run: git diff --exit-code -- backend/.basedpyright/baseline.json
|
||||
- name: pytest
|
||||
run: python -m pytest
|
||||
working-directory: backend
|
||||
# Dependency CVE audit over the pinned lock. BLOCKING since E32: it ran advisory-only, and when
|
||||
# it was finally read it had three urllib3 advisories with a fix available (bumped in the same
|
||||
# change). An advisory without a fix is answered with `--ignore-vuln <id>` here, a reason and a
|
||||
# date in the comment, and a line in docs/gates.md.
|
||||
- name: pip-audit
|
||||
run: pip install --no-cache-dir pip-audit && pip-audit -r backend/requirements.lock
|
||||
@@ -42,3 +42,4 @@ Thumbs.db
|
||||
# TypeScript incremental build artifacts — machine-local, absolute-path-bearing, and no gate lane
|
||||
# reads or regenerates them, so a committed one is stale from the commit that adds it.
|
||||
*.tsbuildinfo
|
||||
frontend/e2e/.speed/
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# The CVE gate's allowlist (E32). The publish gate scans the built image and the committed tree with
|
||||
# The CVE gate's allowlist. The publish gate scans the built image and the committed tree with
|
||||
# Trivy and fails on any HIGH or CRITICAL finding that has a fix available. A finding that cannot be
|
||||
# fixed yet is let through ONLY by an entry here, and every entry carries both:
|
||||
# statement — why it is acceptable for now (what is exposed, what is not)
|
||||
|
||||
@@ -7,6 +7,54 @@ edit the release notes and re-run the generator.
|
||||
|
||||
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
||||
|
||||
## 0.102.0 — 2026-10-08
|
||||
|
||||
### New
|
||||
|
||||
- The library and shared-link players answer the keyboard's and the system's media keys, and resume where you stopped.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Plex player: the browser Back closes the panels over the video one at a time, and only the credits at the very end move on to the next episode. Plus smaller fixes around resume points and stopping a stream.
|
||||
|
||||
## 0.101.0 — 2026-10-06
|
||||
|
||||
### New
|
||||
|
||||
- A shared link plays on any phone, iPhone and iPad included: downloads, clips and uploads are kept in a format every phone plays, and an upload in another format is converted after it arrives.
|
||||
- A shared link opens much faster: its page no longer loads the whole app.
|
||||
|
||||
### Fixed
|
||||
|
||||
- The YouTube player keeps its size on a short window and when maximised — before, it could shrink to a thin strip. Plus smaller fixes around uploads and clips.
|
||||
|
||||
## 0.100.0 — 2026-10-05
|
||||
|
||||
### New
|
||||
|
||||
- Direct messages are gone: the Messages page, the chat dock and their encryption keys were removed with their stored messages, and the privacy policy no longer describes end-to-end encrypted chat.
|
||||
- The shared demo account is gone, with its sign-in entry and its whitelist.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Sign-in security without email: when a Google sign-in takes over an account someone else registered first, or makes it an admin through the admin list, its password is cleared and its sessions end. The owner sets a new one in Settings.
|
||||
- Pressing Back onto a page that no longer exists opens the feed instead of breaking the app.
|
||||
- Behind the scenes: tidier checks and comments across the code.
|
||||
|
||||
## 0.99.2 — 2026-10-04
|
||||
|
||||
### Fixed
|
||||
|
||||
- Settings: the download worker's concurrency and the log level now take effect within seconds — before, the worker ignored both until it was rebuilt.
|
||||
- The YouTube quota history no longer grows forever: the daily cleanup prunes it after a year (Configuration → Quota → Quota history kept).
|
||||
- The live-search button no longer says a search spends the API quota: with the default website source the search itself is free. It names the cost when the official API is selected.
|
||||
|
||||
## 0.99.1 — 2026-10-04
|
||||
|
||||
### Fixed
|
||||
|
||||
- Behind the scenes: every release is now checked by one script on the build machine, including the security audit of every dependency.
|
||||
|
||||
## 0.99.0 — 2026-10-04
|
||||
|
||||
### New
|
||||
|
||||
@@ -13,15 +13,15 @@ router-free navigation provider.
|
||||
## The gate — never merge or ship red
|
||||
|
||||
The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + the lock check + typecheck (3 tsc projects) +
|
||||
eslint + prettier --check + knip (both modes) + vitest + vite build + pytest**, then a coverage report (no threshold). It runs
|
||||
eslint + prettier --check + knip + vitest + vite build + pytest + pip-audit + npm audit**. It runs
|
||||
before every `release` merges and on every `publish`, which **refuses to build on failure — there is deliberately no `--skip`
|
||||
flag**; a release also requires the Playwright suite, and the image must pass a Trivy scan before it is pushed. Run it and get it
|
||||
green **before** any merge, release, or handing work to review.
|
||||
|
||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||
- Directly, without the dispatcher: the commands in [`docs/development.md`](docs/development.md) § The gate (frontend lanes, `ruff` over `backend scripts`, basedpyright in the test image, and `pytest` — the DB-lane needs Postgres).
|
||||
- **A gate with any real hit is never switched off or narrowed** — not a lint rule, a compiler flag, a lane or a release step. Fix the sites, or suppress each one inline naming its invariant (`-- why`, lint-enforced). What stays narrowed is in [`docs/gates.md`](docs/gates.md), and a contract test fails when a narrowing is missing from it. The effective ESLint config of every file and the CI/lane lines are pinned in `frontend/src/lib/gates.snapshot.json`: a new code file or any gate change fails until you regenerate it (`GATE_SNAPSHOT=update`, see gates.md) — then read that diff as a gate change. A comment saying "X is enforced by Y" is checked by running Y (E32: one such comment hid 95 functions the React Compiler skipped).
|
||||
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` and burn down epic by epic. Fixed some? Regenerate with `--writebaseline`.
|
||||
- Full gate: `bash scripts/check.sh` (`lint` for the static lanes only); the private ops dispatcher's `siftlode check` runs the same script. Single lanes: [`docs/development.md`](docs/development.md) § The gate.
|
||||
- **There is no CI** (owner, 2026-10-04): this gate, run here, is the only one — green before every hand-over, and `release` refuses to merge without it.
|
||||
- **A gate with any real hit is never switched off or narrowed** — not a lint rule, a compiler flag, a lane or a release step. Fix the sites, or suppress each one inline naming its invariant (`-- why`, lint-enforced). What stays narrowed is in [`docs/gates.md`](docs/gates.md), and a contract test fails when a narrowing is missing from it. The effective ESLint config of every scope and the gate/lane lines are pinned in `frontend/src/lib/gates.snapshot.json`: any gate change fails until you regenerate it (`GATE_SNAPSHOT=update`, see gates.md) — then read that diff as a gate change. A comment saying "X is enforced by Y" is checked by running Y (E32: one such comment hid 95 functions the React Compiler skipped).
|
||||
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` (no burn-down mandate, but none was ever triaged: a `reportOptional*` entry in code you touch may be a real bug). A run that fixed some rewrites it — commit that.
|
||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||
|
||||
## Workflow
|
||||
@@ -63,9 +63,10 @@ how it gets built. Concretely:
|
||||
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`,
|
||||
`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → module
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell (a shared `/watch` link boots from its own
|
||||
entry, `watch.html` → `watchMain.tsx`, held app-free by a contract test) → `moduleRegistry` route-chunks → module
|
||||
components → the **platform layer `components/ui/`** (domain-free primitives: the popover engine and
|
||||
its menus, `Modal`/`Overlay`, `Button`/`form`, `DataTable`/`VirtualGrid`, `QueryState`, …). That
|
||||
boundary is one-way and **lint-enforced**, not just described: a platform component may not import a
|
||||
@@ -95,17 +96,13 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE button | `ui/Button` (a `SIZES` rung) · `IconButton` · `Pressable` (a clickable card/row) · `Chip` · `DragHandle` | a raw `<button>` or `FocusSafeButton` outside the control primitives (lint-enforced) |
|
||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||
| FE text field / checkbox / slider | `ui/Input` · `Textarea` · `InnerInput` in a `fieldClass` frame · `Checkbox`/`CheckRow` · `Slider` | a raw `<input>`/`<textarea>` anywhere but those primitives (lint-enforced) |
|
||||
| FE glass surface | a role: `SURFACE.panel` / `card` / `menu` / `popover` / `tooltip` (`ui/surfaces.ts`) | a bare `glass`/`glass-card`/`glass-menu` beside a hand-picked radius (lint-enforced) |
|
||||
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||
| FE player on a native media element | `lib/mediaSession` (`useMediaSession`) + `lib/mediaKeys` + `lib/resume` (`useResume`), the element held as STATE | a private `navigator.mediaSession` / `keydown` / position-saving effect per player |
|
||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
||||
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
||||
@@ -114,6 +111,11 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
| FE popover / scroll-fade ref | destructure it: `const { setReference } = pop.refs`, then `ref={setReference}` | `ref={pop.refs.setReference}` / `ref={fade.ref}` — the compiler then treats the whole object as a ref and skips |
|
||||
| FE async handler with cleanup | `try { … } catch { … }`, then the cleanup after it (steps that `return` early go in their own function); a conditional, loop or `?.` the `try` needs goes before it or into a module-level helper | `try/finally`, or a `?:` / `??` / loop / `?.` inside `try` — the React Compiler skips the whole component (and `react-hooks/todo` misses most of these) |
|
||||
|
||||
Not repeated here, because a lint rule refuses the wrong one and its message names the right one:
|
||||
buttons (`ui/Button`, `IconButton`, `Pressable`, `Chip`, `DragHandle`), text fields and native
|
||||
controls (`ui/Input`, `Checkbox`, `Slider`, …), glass surfaces (`SURFACE.*`), a native `<select>`
|
||||
(`SegmentedToggle` / `FilterSelect`), and network calls (`req` / `uploadReq` / `beacon`, `lib/http.ts`).
|
||||
|
||||
## House rules that bite (learned from real bugs)
|
||||
|
||||
- **A rule stated in a comment must also be enforced in code.** The worst bugs here came from a comment
|
||||
@@ -125,7 +127,8 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
||||
"no results". Reads degrade honestly; writes are conditional (`UPDATE … WHERE status='running'`).
|
||||
- **i18n parity.** HU/EN, zero key drift. Every user-facing string is translated.
|
||||
- **Comment the "why", keep it current.** This codebase uses comments as its architecture doc; a stale
|
||||
"why" is a liability, so update it when the behavior changes.
|
||||
"why" is a liability, so update it when the behavior changes. Say the why itself, never the plan
|
||||
step that produced it: no epic, sprint or review-round ids in the code (`planCodes.contract.test.ts`).
|
||||
|
||||
---
|
||||
|
||||
|
||||
+6
-6
@@ -11,7 +11,7 @@ ARG BUILD_DATE=
|
||||
|
||||
# Stage 1: build the frontend SPA
|
||||
# The major is pinned to `.nvmrc` and held there by frontend/src/lib/nodeEngine.contract.test.ts:
|
||||
# this image, the CI gate's image and the dev machine must be ONE engine (see that test for why).
|
||||
# this image and the dev machine the gate runs on must be ONE engine (see that test for why).
|
||||
FROM node:24-alpine AS frontend
|
||||
WORKDIR /fe
|
||||
# Deps first, keyed on the lockfile, so editing app source never re-resolves node_modules.
|
||||
@@ -47,13 +47,13 @@ RUN adduser --disabled-password --gecos "" appuser \
|
||||
&& mkdir -p /downloads \
|
||||
&& chown appuser /app /downloads
|
||||
|
||||
# ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode (E6 S1).
|
||||
# ffmpeg: merge separate video+audio streams + postprocessing, and the Plex VAAPI transcode.
|
||||
# intel-media-va-driver (iHD): the VA driver ffmpeg's h264_vaapi needs — libva2 ships in the base but
|
||||
# no driver, so without this `vainfo`/`h264_vaapi` fail and transcode silently falls back to libx264.
|
||||
# Harmless on hosts with no /dev/dri (the driver just goes unused). Deno: the JavaScript runtime yt-dlp
|
||||
# uses to solve YouTube's "n" signature challenge for the web player clients (required alongside the PO
|
||||
# token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
||||
# `apt-get upgrade` (E32): the base image is a snapshot, and the Debian security fixes released since it
|
||||
# `apt-get upgrade`: the base image is a snapshot, and the Debian security fixes released since it
|
||||
# was cut are only in the archive — the first Trivy scan found one (pcre2) fixed upstream but not in it.
|
||||
ARG DENO_VERSION=v2.9.1
|
||||
RUN apt-get update \
|
||||
@@ -79,9 +79,9 @@ RUN printf 'Types: deb\nURIs: http://deb.debian.org/debian\nSuites: bookworm\nCo
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install from the PINNED lock (requirements.lock, generated from requirements.txt) so the image is
|
||||
# byte-reproducible and the exact versions are the ones pip-audit scanned in CI — not whatever floats
|
||||
# up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
||||
# pip itself is removed once it has installed the lock (E32): nothing runs it at runtime, and it carries
|
||||
# byte-reproducible and the exact versions are the ones the gate's pip-audit scanned — not whatever
|
||||
# floats up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
||||
# pip itself is removed once it has installed the lock: nothing runs it at runtime, and it carries
|
||||
# its own VENDORED copies of urllib3, msgpack and setuptools, which Trivy reports as the image's own —
|
||||
# an attack surface and a CVE stream for a tool the running app never calls.
|
||||
COPY backend/requirements.lock .
|
||||
|
||||
@@ -23,8 +23,9 @@ blocker and SponsorBlock keep working.
|
||||
- **Playlists with two-way YouTube sync** — build them locally, keep them in sync in both directions.
|
||||
- **In-app player** with resume, plus keyboard/scroll controls.
|
||||
- **Download Center** — save videos to the server with yt-dlp in a Plex-friendly layout (format
|
||||
presets, per-user storage quota), trim / crop / split & join them in a built-in editor, then save
|
||||
to your device, share with another user, or hand out a public watch link.
|
||||
presets, per-user storage quota), trim / crop / split & join them in a built-in editor, play them
|
||||
where you left off, then save to your device, share with another user, or hand out a public watch
|
||||
link.
|
||||
- **Plex library** *(optional)* — point Siftlode at your own Plex server to browse and play its
|
||||
movies and shows alongside your feed, with watch state kept in sync.
|
||||
- **Multi-user** with per-user private state, a shared catalog, and a fair daily API-quota guard.
|
||||
|
||||
@@ -743,14 +743,6 @@
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 37,
|
||||
"endColumn": 47,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
@@ -824,96 +816,6 @@
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/messages.py": [
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 11,
|
||||
"endColumn": 43,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 23,
|
||||
"endColumn": 27,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 13,
|
||||
"endColumn": 5,
|
||||
"lineCount": 7
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 8,
|
||||
"endColumn": 15,
|
||||
"lineCount": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportReturnType",
|
||||
"range": {
|
||||
"startColumn": 15,
|
||||
"endColumn": 72,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportCallIssue",
|
||||
"range": {
|
||||
"startColumn": 30,
|
||||
"endColumn": 48,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 41,
|
||||
"endColumn": 44,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 36,
|
||||
"endColumn": 38,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 71,
|
||||
"endColumn": 73,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportArgumentType",
|
||||
"range": {
|
||||
"startColumn": 30,
|
||||
"endColumn": 39,
|
||||
"lineCount": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"code": "reportOptionalMemberAccess",
|
||||
"range": {
|
||||
"startColumn": 27,
|
||||
"endColumn": 29,
|
||||
"lineCount": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"./app/routes/playlists.py": [
|
||||
{
|
||||
"code": "reportOptionalOperand",
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
"""messages: drop direct messaging (the owner's decision, 2026-10-04)
|
||||
|
||||
Chat — end-to-end encrypted direct messages plus the server's welcome note — was never used: prod held
|
||||
two welcome notes, no message between people, and one key. Its tables go with its code. The rows are
|
||||
not migrated anywhere; the nightly backup holds them.
|
||||
|
||||
The downgrade restores the schema as it stood (0026 + 0027), empty: a message can't be brought back
|
||||
from a migration, and the E2EE key blobs were only ever readable by their owner's browser anyway.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0097_drop_messages"
|
||||
down_revision: str | None = "0096_videos_sort_indexes"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_table("message_keys")
|
||||
op.drop_index("ix_messages_created_at", table_name="messages")
|
||||
op.drop_index("ix_messages_recipient_id", table_name="messages")
|
||||
op.drop_index("ix_messages_sender_id", table_name="messages")
|
||||
op.drop_table("messages")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.create_table(
|
||||
"messages",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"sender_id", sa.Integer(), sa.ForeignKey("users.id", ondelete="CASCADE"), nullable=True
|
||||
),
|
||||
sa.Column(
|
||||
"recipient_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("body", sa.Text(), nullable=True),
|
||||
sa.Column("read_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.Column("kind", sa.String(length=16), nullable=False, server_default="user"),
|
||||
sa.Column("ciphertext", sa.Text(), nullable=True),
|
||||
sa.Column("iv", sa.String(length=32), nullable=True),
|
||||
)
|
||||
op.create_index("ix_messages_sender_id", "messages", ["sender_id"])
|
||||
op.create_index("ix_messages_recipient_id", "messages", ["recipient_id"])
|
||||
op.create_index("ix_messages_created_at", "messages", ["created_at"])
|
||||
op.create_table(
|
||||
"message_keys",
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
),
|
||||
sa.Column("public_key", sa.Text(), nullable=False),
|
||||
sa.Column("wrapped_private_key", sa.Text(), nullable=False),
|
||||
sa.Column("salt", sa.String(length=64), nullable=False),
|
||||
sa.Column("wrap_iv", sa.String(length=32), nullable=False),
|
||||
sa.Column("key_check", sa.Text(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,55 @@
|
||||
"""demo: drop the shared demo account (the owner's decision, 2026-10-04)
|
||||
|
||||
The demo account — one shared user that whitelisted emails entered from the login page, reset on a
|
||||
schedule — goes with its code. Prod held the demo user and 10 whitelist rows.
|
||||
|
||||
The upgrade deletes the demo user first (its playlists, states and subscriptions cascade; audit and
|
||||
quota rows keep their history with the actor nulled, as for any deleted account), then the traces
|
||||
that only the demo code could render: its audit rows (whitelist edits, manual resets, and every row
|
||||
about the reset job — its runs and its interval changes), the reset job's interval override and its
|
||||
completion notices. Nothing is migrated anywhere; the nightly backup holds the rows.
|
||||
|
||||
The downgrade restores the schema as it stood (0014), empty: the demo user is created lazily on its
|
||||
first sign-in, so an empty `demo_whitelist` is the whole of its state.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0098_drop_demo_account"
|
||||
down_revision: str | None = "0097_drop_messages"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("DELETE FROM users WHERE is_demo")
|
||||
op.execute(
|
||||
"DELETE FROM audit_log WHERE action IN ('demo_add', 'demo_remove', 'demo_reset')"
|
||||
" OR (target_type = 'job' AND target_id = 'demo_reset')"
|
||||
)
|
||||
op.execute("DELETE FROM scheduler_settings WHERE job_id = 'demo_reset'")
|
||||
op.execute("DELETE FROM notifications WHERE data->>'job_id' = 'demo_reset'")
|
||||
op.drop_index("ix_demo_whitelist_email", table_name="demo_whitelist")
|
||||
op.drop_table("demo_whitelist")
|
||||
op.drop_index("ix_users_is_demo", table_name="users")
|
||||
op.drop_column("users", "is_demo")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column(
|
||||
"users",
|
||||
sa.Column("is_demo", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
)
|
||||
op.create_index("ix_users_is_demo", "users", ["is_demo"])
|
||||
op.create_table(
|
||||
"demo_whitelist",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("email", sa.String(length=320), nullable=False),
|
||||
sa.Column("note", sa.Text(), nullable=True),
|
||||
sa.Column("added_by", sa.String(length=320), nullable=True),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
)
|
||||
op.create_index("ix_demo_whitelist_email", "demo_whitelist", ["email"], unique=True)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""library: a resume point per user and library item
|
||||
|
||||
The library player opened every item at 0:00. `library_states` keeps where a user stopped, per job
|
||||
(their own download, or one shared with them) — the same shape as `video_states`' position, without a
|
||||
status: the library has no "watched". It starts empty; the downgrade drops it.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0099_library_states"
|
||||
down_revision: str | None = "0098_drop_demo_account"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"library_states",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"job_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("download_jobs.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("position_seconds", sa.Integer(), nullable=False),
|
||||
sa.Column("progress_updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),
|
||||
)
|
||||
op.create_index("ix_library_states_job_id", "library_states", ["job_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_library_states_job_id", table_name="library_states")
|
||||
op.drop_table("library_states")
|
||||
@@ -0,0 +1,22 @@
|
||||
"""plex_items: drop `markers`, a column nothing writes or reads
|
||||
|
||||
Intro/credits markers are read live from Plex for the item the player opens (with Plex's `final` flag
|
||||
on the block that ends it); this cached copy was never filled — every row is NULL. The downgrade
|
||||
restores the empty column.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0100_drop_plex_item_markers"
|
||||
down_revision: str | None = "0099_library_states"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_column("plex_items", "markers")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column("plex_items", sa.Column("markers", sa.JSON(), nullable=True))
|
||||
@@ -1,7 +1,7 @@
|
||||
# Backend conventions
|
||||
|
||||
The rules below are already followed across `backend/app` — they lived only in a few module
|
||||
docstrings (`demo.py`, `userscope.py`, `routes/_common.py`) until this file collected them (E3 S3,
|
||||
docstrings (`userscope.py`, `routes/_common.py`) until this file collected them (E3 S3,
|
||||
B-core-20). When you add code, match these so the next reader doesn't have to guess "which pattern do
|
||||
I copy".
|
||||
|
||||
@@ -21,7 +21,7 @@ Three layers, by *who* runs the code — not by feature:
|
||||
`issues/bundle.py` is the one definition of a bug report's files, for the zip export and the tracker.
|
||||
|
||||
3. **Flat `app/*.py` core modules — cross-cutting primitives.** `auth`, `sysconfig`, `quota`, `audit`,
|
||||
`notifications`, `userscope`, `realtime`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
`notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging
|
||||
configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log
|
||||
level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the
|
||||
reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core
|
||||
|
||||
@@ -27,7 +27,7 @@ class AuditAction:
|
||||
USER_SUSPEND = "user_suspend"
|
||||
USER_UNSUSPEND = "user_unsuspend"
|
||||
USER_DELETE = "user_delete"
|
||||
# E9.5 S5: an admin let a stuck registrant in, or re-sent their confirmation link. Both are
|
||||
# An admin let a stuck registrant in, or re-sent their confirmation link. Both are
|
||||
# recorded because both are the admin acting ON someone's access, not merely reading a list.
|
||||
USER_ACTIVATE = "user_activate"
|
||||
USER_VERIFY_RESEND = "user_verify_resend"
|
||||
@@ -36,12 +36,9 @@ class AuditAction:
|
||||
INVITE_DENY = "invite_deny"
|
||||
INVITE_ADD = "invite_add"
|
||||
INVITE_DELETE = "invite_delete"
|
||||
# E9.5 S6: the admin chose to tell someone their address is whitelisted. Adding the address is
|
||||
# The admin chose to tell someone their address is whitelisted. Adding the address is
|
||||
# silent, so this is the act that actually reaches a person — and the one worth a record.
|
||||
INVITE_SEND = "invite_send"
|
||||
DEMO_ADD = "demo_add"
|
||||
DEMO_REMOVE = "demo_remove"
|
||||
DEMO_RESET = "demo_reset"
|
||||
DISCOVERY_PURGE = "discovery_purge"
|
||||
# Config
|
||||
CONFIG_SET = "config_set"
|
||||
@@ -58,7 +55,7 @@ class AuditAction:
|
||||
DOWNLOAD_CREDENTIAL_DELETE = "download_credential_delete"
|
||||
# The log itself was cleared (a single row survives the clear as tamper-evidence)
|
||||
AUDIT_CLEAR = "audit_clear"
|
||||
# E16: a user sent a debug session's issue to the tracker — the one act that takes their log
|
||||
# A user sent a debug session's issue to the tracker — the one act that takes their log
|
||||
# off this instance, so it is recorded (target = the session, summary = the issue's URL).
|
||||
ISSUE_CREATE = "issue_create"
|
||||
|
||||
|
||||
+10
-47
@@ -24,8 +24,6 @@ def operator_contact() -> str | None:
|
||||
return next(iter(sorted(settings.admin_email_set)), None)
|
||||
|
||||
|
||||
DEMO_GOOGLE_SUB = "__demo__"
|
||||
DEMO_EMAIL = "demo@siftlode.local"
|
||||
MAX_SESSION_ACCOUNTS = 6
|
||||
|
||||
# OAuth scope vocabulary — the atoms + the request bundles, kept together as core. has_*_scope check
|
||||
@@ -69,7 +67,7 @@ def has_write_scope(user: User) -> bool:
|
||||
def env_allowed_canonicals() -> set[str]:
|
||||
"""The env bootstrap allow-list, in CANONICAL form — the only shape it may be compared in.
|
||||
|
||||
S3 canonicalised the Invite branch of `is_allowed` and left this one matching the literal string,
|
||||
Canonicalising only the Invite branch of `is_allowed` left this one matching the literal string,
|
||||
which is the 2026-09-03 lockout with a different source of truth: an operator who writes
|
||||
`first.last@gmail.com` into ALLOWED_EMAILS is refused when Google answers with the undotted
|
||||
primary, and a `+tag`ged entry can never match at all, because a Google account cannot contain a
|
||||
@@ -129,7 +127,7 @@ def can_activate(db: Session, user: User) -> bool:
|
||||
Lives in core so the admin surface and the /api/me badge count answer from ONE definition; a
|
||||
button that appears and a number that counts must never disagree about who is stuck.
|
||||
"""
|
||||
if user.is_demo or user.is_suspended or user.is_active:
|
||||
if user.is_suspended or user.is_active:
|
||||
return False
|
||||
return is_allowed(db, user.email)
|
||||
|
||||
@@ -147,7 +145,6 @@ def count_pending_activations(db: Session) -> int:
|
||||
rows = (
|
||||
db.execute(
|
||||
select(User).where(
|
||||
User.is_demo.is_(False),
|
||||
User.is_suspended.is_(False),
|
||||
User.is_active.is_(False),
|
||||
)
|
||||
@@ -188,24 +185,6 @@ def find_user_by_email(db: Session, email: str) -> User | None:
|
||||
return candidates[0] if len(candidates) == 1 else None
|
||||
|
||||
|
||||
def get_or_create_demo_user(db: Session) -> User:
|
||||
"""The one shared demo user, created on first use. No OAuth token / YouTube scope, so
|
||||
has_read_scope/has_write_scope are False and every YouTube-touching path is closed to it."""
|
||||
user = db.query(User).filter(User.is_demo.is_(True)).one_or_none()
|
||||
if user is None:
|
||||
user = User(
|
||||
google_sub=DEMO_GOOGLE_SUB,
|
||||
email=DEMO_EMAIL,
|
||||
display_name="Demo",
|
||||
role="user",
|
||||
is_demo=True,
|
||||
preferences={"language": "en"},
|
||||
)
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return user
|
||||
|
||||
|
||||
def revoke_google_token(token: str | None) -> None:
|
||||
"""Best-effort revocation of a Google OAuth grant at Google's endpoint. Revoking the refresh
|
||||
token tears down the whole grant, so a later sign-in starts from a clean consent (no scopes
|
||||
@@ -278,8 +257,7 @@ ACTIVE_ACCOUNT_HEADER = "x-siftlode-account"
|
||||
|
||||
def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int | None, bool]:
|
||||
"""Which account this connection acts as, and whether that's the session's default account.
|
||||
Takes any HTTPConnection — an HTTP Request OR a WebSocket — so the WS push channel shares this
|
||||
exact per-tab resolution instead of re-implementing it.
|
||||
Takes an HTTPConnection rather than a Request: it reads only the session and the headers.
|
||||
|
||||
The signed cookie holds the *wallet* (`account_ids`, every account signed into this browser)
|
||||
plus a default `user_id`. A caller may name its account with the X-Siftlode-Account header (or
|
||||
@@ -297,8 +275,8 @@ def resolved_user_id(conn: HTTPConnection, *, strict: bool = True) -> tuple[int
|
||||
"""
|
||||
default_id = conn.session.get("user_id")
|
||||
wallet = conn.session.get("account_ids") or []
|
||||
# Header for normal XHR; ?account= for contexts that can't set headers (WebSocket, and a
|
||||
# plain <a> file download). Both are wallet-gated below.
|
||||
# Header for normal XHR; ?account= for a context that can't set headers (a plain <a> file
|
||||
# download). Both are wallet-gated below.
|
||||
hdr = conn.headers.get(ACTIVE_ACCOUNT_HEADER) or conn.query_params.get("account")
|
||||
if hdr:
|
||||
try:
|
||||
@@ -362,12 +340,12 @@ def _session_user(request: Request, db: Session, user_id: int | None, is_default
|
||||
if default_id and default_id not in accounts:
|
||||
accounts.append(default_id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
# Attribute this request's log lines to the account (E22). Here, not in the request middleware:
|
||||
# Attribute this request's log lines to the account. Here, not in the request middleware:
|
||||
# this is the one place a session becomes an account, and the middleware runs outside both the
|
||||
# session middleware and this task's context. `request.state` is shared across middleware layers
|
||||
# (it lives on the ASGI scope), so the access-log line can read it back. The user's consented
|
||||
# diagnostics level rides along: their requests are logged — and stored — in that detail (S5).
|
||||
# So does a debug session the request names (E16): only the caller's own open one counts, and a
|
||||
# diagnostics level rides along: their requests are logged — and stored — in that detail.
|
||||
# So does a debug session the request names: only the caller's own open one counts, and a
|
||||
# header naming anything else is ignored rather than refused — a stale tab must keep working.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, until = debugsession.level_for(user, session)
|
||||
@@ -403,25 +381,10 @@ def optional_current_user(request: Request, db: Session = Depends(get_db)) -> Us
|
||||
return None
|
||||
|
||||
|
||||
def require_human(user: User = Depends(current_user)) -> User:
|
||||
"""Reject the shared demo account from actions that need a real Google/YouTube identity
|
||||
or spend the shared quota. Most YouTube paths are already closed to it implicitly (it has
|
||||
no token, so has_read_scope/has_write_scope are False); this makes the boundary explicit
|
||||
where there's no scope check to lean on."""
|
||||
if user.is_demo:
|
||||
# Structured, because `req` leaves 403 to the caller and `notifyActionError` only shows a
|
||||
# STRUCTURED detail — a plain string here reached the demo user as their caller's generic
|
||||
# "couldn't do that", with no hint that the demo account is the reason (round 6). Affordances
|
||||
# this gate sits behind are not all UI-gated on `is_demo` (the comfort shelf is not).
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
return user
|
||||
|
||||
|
||||
def require_write_scope(user: User = Depends(require_human)) -> User:
|
||||
def require_write_scope(user: User = Depends(current_user)) -> User:
|
||||
"""Dependency: require the YouTube write scope (subscribe/unsubscribe, playlist export/edit).
|
||||
Single source of truth for the write gate — replaces the inline `if not has_write_scope(user)`
|
||||
checks that had drifted into three different 403 messages across channels/playlists. Layers on
|
||||
`require_human` because writing to YouTube needs a real, non-demo identity."""
|
||||
checks that had drifted into three different 403 messages across channels/playlists."""
|
||||
if not has_write_scope(user):
|
||||
raise refused(403, "youtube_editing_off")
|
||||
return user
|
||||
|
||||
@@ -59,9 +59,9 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024
|
||||
# a say.
|
||||
#
|
||||
# Both listed routes now satisfy that. The screenshot route always did (`capped_body` streams it,
|
||||
# and the owner check resolves first). The upload route did NOT until P1: it declared
|
||||
# and the owner check resolves first). The upload route did NOT at first: it declared
|
||||
# `file: UploadFile = File(...)`, so an anonymous caller could make the process write the whole
|
||||
# per-file allowance to temp before `require_human` refused them — this middleware bounded that
|
||||
# per-file allowance to temp before the auth dependency refused them — this middleware bounded that
|
||||
# number but could not stop the writing. It now reads its own body from a dependency declared after
|
||||
# the auth one, so its in-handler cap measures bytes as they arrive, like everything else here.
|
||||
#
|
||||
@@ -73,12 +73,12 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024
|
||||
# `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the
|
||||
# default, so listing it could only widen what it accepts.
|
||||
ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
||||
# Ingesting a local file. The body IS the file since P1 — no multipart envelope to allow for —
|
||||
# Ingesting a local file. The body IS the file now — no multipart envelope to allow for —
|
||||
# so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s
|
||||
# `download_upload_max_bytes` is bounded by this same constant (see sysconfig.SPECS), so an
|
||||
# admin can only ever LOWER the route's cap below the edge, never raise it past one.
|
||||
#
|
||||
# WHICH LAYER ANSWERS, measured rather than assumed (deep-review round 1):
|
||||
# WHICH LAYER ANSWERS, measured rather than assumed:
|
||||
# * knob LOWERED (an override below the default) — the route's cap is the smaller number, so
|
||||
# `capped_stream_to` trips first and answers 422 `upload_too_large` carrying the admin's own
|
||||
# figure, after draining the rest of the body so the client can read it.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Alphabetical order in the reader's language (E30 S3).
|
||||
"""Alphabetical order in the reader's language.
|
||||
|
||||
Why not the database default: the Postgres image is Alpine, and musl's `en_US.utf8` collation is BYTE
|
||||
order — every accented initial sorted after "Z" ("Zorró < Életrajzi < Ötödik"), in every A→Z list the
|
||||
|
||||
+11
-10
@@ -146,8 +146,6 @@ class Settings(BaseSettings):
|
||||
autotag_interval_minutes: int = 30
|
||||
subscriptions_resync_minutes: int = 360
|
||||
playlist_sync_minutes: int = 360
|
||||
# How often the shared demo account is auto-reset to a clean baseline (communal sandbox).
|
||||
demo_reset_minutes: int = 720
|
||||
# --- Maintenance / validation job ---
|
||||
# Runs once a day by default (admin-tunable via the Scheduler dashboard). Grace periods
|
||||
# are in days because that's the granularity that matters; the rolling re-validation
|
||||
@@ -244,7 +242,7 @@ class Settings(BaseSettings):
|
||||
download_default_max_bytes: int = 5_368_709_120 # 5 GiB
|
||||
download_default_max_concurrent: int = 2
|
||||
download_default_max_jobs: int = 50
|
||||
# Per-file cap for local uploads (E7). Unlike a download (whose size is unknown until the worker
|
||||
# Per-file cap for local uploads. Unlike a download (whose size is unknown until the worker
|
||||
# finishes), an upload's size is known at ingest, so this bounds a single file directly; the
|
||||
# user's overall byte footprint is still governed by download_default_max_bytes.
|
||||
# ge=1: this number is also the EDGE ceiling (`bodylimit.ROUTE_MAX_BODY_BYTES`), where 0 is a
|
||||
@@ -260,12 +258,15 @@ class Settings(BaseSettings):
|
||||
# Admin audit log: how long to keep rows (days; 0 = keep forever) + how often the GC runs.
|
||||
audit_retention_days: int = 180
|
||||
audit_gc_minutes: int = 1440
|
||||
# YouTube quota history (quota_events): days kept (0 = forever). A year, so the admin usage view's
|
||||
# longest window (365 days) stays whole; the same daily cleanup job prunes it.
|
||||
quota_retention_days: int = 365
|
||||
# On-disk layout: "plex" (Channel/Season YYYY/… [id].ext + .nfo/poster) or "flat".
|
||||
download_layout: DownloadLayout = "plex"
|
||||
# Default SponsorBlock (skip sponsor segments) for new custom profiles.
|
||||
sponsorblock_default: bool = False
|
||||
|
||||
# --- Diagnostics / logging (E22) ---
|
||||
# --- Diagnostics / logging ---
|
||||
# BOOTSTRAP level only: the running level is the `log_level` admin knob (sysconfig), applied at
|
||||
# startup and on change without a restart. This is what a process logs at before it can reach
|
||||
# the database — and the only knob the worker/runner children read, since they are short-lived.
|
||||
@@ -298,7 +299,7 @@ class Settings(BaseSettings):
|
||||
plex_path_map: str = ""
|
||||
# Which Plex library section keys are exposed (comma-separated). Empty = all enabled sections.
|
||||
plex_libraries: str = ""
|
||||
# HU-overlay mapping (E5 S5): comma-separated `huSectionKey:enSectionKey` pairs. Each named HU
|
||||
# HU-overlay mapping: comma-separated `huSectionKey:enSectionKey` pairs. Each named HU
|
||||
# section is synced as a non-browsable overlay whose Hungarian metadata is written onto the EN
|
||||
# twin section's rows (matched by guid). Empty = no overlays. E.g. "3:4,5:1".
|
||||
plex_overlay_map: str = ""
|
||||
@@ -308,7 +309,7 @@ class Settings(BaseSettings):
|
||||
plex_sync_interval_min: int = 360
|
||||
# How often the incremental Plex→Siftlode watch-state reconcile runs, in minutes (Phase C):
|
||||
# a cheap history + on-deck pull plus a re-push of any states that failed their immediate push.
|
||||
# 15 min (S2): with Plex now the source of truth (prompt write-back on the ~10s checkpoint +
|
||||
# 15 min: with Plex now the source of truth (prompt write-back on the ~10s checkpoint +
|
||||
# Plex-authoritative pull), a faster incremental pull keeps Siftlode close to Plex-side changes.
|
||||
plex_watch_sync_interval_min: int = 15
|
||||
# How often the FULL watch-state reconcile runs, in minutes (Phase C). A whole-section rescan
|
||||
@@ -323,7 +324,7 @@ class Settings(BaseSettings):
|
||||
# each one is a whole segmented copy of the film on the scratch volume, written faster than
|
||||
# realtime. One account holds at most two regardless (plex/stream.py `_MAX_SESSIONS_PER_USER`).
|
||||
plex_max_sessions: int = 8
|
||||
# --- Transcode (playable="transcode") re-encode tuning (E6 S1) ---
|
||||
# --- Transcode (playable="transcode") re-encode tuning ---
|
||||
# Hardware-encoder policy: "auto" = VAAPI (h264_vaapi) when the iGPU render node is present AND the
|
||||
# source codec is hardware-decodable, else libx264 software; "on" = force VAAPI wherever the render
|
||||
# node exists (skip the decodability gate); "off" = always software. See plex/stream.py `_select_encode`.
|
||||
@@ -346,7 +347,7 @@ class Settings(BaseSettings):
|
||||
# local disk, so the default is fine there.
|
||||
plex_hls_dir: str = ""
|
||||
|
||||
# --- TMDB enrichment (E5 S2; optional — enriches the Plex catalog from TMDB) ---
|
||||
# --- TMDB enrichment (optional — enriches the Plex catalog from TMDB) ---
|
||||
# TMDB API key (v3 api_key, 32 hex chars). Server-side only; stored encrypted when set via the
|
||||
# admin Config page. Empty → the TMDB enrichment job no-ops. Feeds keywords / crew / budget /
|
||||
# revenue / TMDb score / recommendations / backdrops onto the mirrored Plex rows, matched by the
|
||||
@@ -356,7 +357,7 @@ class Settings(BaseSettings):
|
||||
# Metadata language for TMDB fetches (keyword/overview locale). Stored EN by default.
|
||||
tmdb_language: str = "en-US"
|
||||
|
||||
# --- Bug reports (E16): where a debug session's issue goes. All three empty = no tracker; the
|
||||
# --- Bug reports: where a debug session's issue goes. All three empty = no tracker; the
|
||||
# composer still exports a zip. The repository must be PRIVATE — an issue carries the reporter's
|
||||
# log — and app/issues/gitea.py refuses a public one. Runtime knobs (sysconfig, admin UI). ---
|
||||
issues_gitea_url: str = ""
|
||||
@@ -366,7 +367,7 @@ class Settings(BaseSettings):
|
||||
tmdb_enrich_interval_min: int = 120
|
||||
# Titles fetched per TMDB enrichment run (TMDB is fast + generously rate-limited).
|
||||
tmdb_enrich_batch: int = 100
|
||||
# How often the people job (E30 S6) fetches a batch of person bios from TMDB and portraits through Plex,
|
||||
# How often the people job fetches a batch of person bios from TMDB and portraits through Plex,
|
||||
# in minutes, and how many of each per run: ~18k linked people at 300 per 10 min is about ten hours
|
||||
# once, then only the newcomers.
|
||||
people_enrich_interval_min: int = 10
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, and the evidence a bug report is built from.
|
||||
"""Debug sessions: Start → reproduce → Stop, and the evidence a bug report is built from.
|
||||
|
||||
While a session is open its user's browser names it in `X-Debug-Session` on every request. The auth
|
||||
dependency accepts the header only for the caller's OWN open session (anything else is ignored, never
|
||||
@@ -269,7 +269,7 @@ def set_delivery(db: Session, session: DebugSession, state: dict, token: str) ->
|
||||
every step. It is stamped from `token` here instead, and only after the ROW confirms the claim
|
||||
is still ours. Without that check the write is a full-column overwrite with no WHERE: a send
|
||||
that lost its lease came back and stomped the take-over send's committed links, then put its
|
||||
own dead token back on an idle session (deep-review round 4). `release_send` has always made
|
||||
own dead token back on an idle session. `release_send` has always made
|
||||
the same check; this is the other half of it."""
|
||||
if claim_owner(db, session) != token:
|
||||
raise SendClaimLost(session.id)
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
"""Demo-account service: reset the shared demo sandbox to a clean, seeded baseline.
|
||||
|
||||
Shared by the admin's manual reset endpoint (routes/admin.py) and the scheduled demo-reset job
|
||||
(scheduler.py). Lives in the service layer, not routes, so the scheduler can call it without an
|
||||
`app.routes -> app.scheduler` import cycle (the reason its import used to be lazy).
|
||||
"""
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Playlist, PlaylistItem, Subscription, User, Video, VideoState
|
||||
|
||||
|
||||
def _seed_demo_playlists(db: Session, demo: User) -> int:
|
||||
"""Seed a few sample playlists from the shared catalog so a freshly-reset demo isn't
|
||||
empty — gives visitors something to play with. Best-effort: skips silently if the
|
||||
catalog is too small."""
|
||||
recent = (
|
||||
db.execute(
|
||||
select(Video.id)
|
||||
.where(Video.is_short.is_(False), Video.live_status == "none")
|
||||
.order_by(Video.published_at.desc().nullslast())
|
||||
.limit(60)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not recent:
|
||||
return 0
|
||||
samples = [
|
||||
("Fresh picks", recent[0:10]),
|
||||
("Quick watch", recent[10:18]),
|
||||
("Saved for later", recent[18:26]),
|
||||
]
|
||||
created = 0
|
||||
for pos, (name, vids) in enumerate(samples):
|
||||
if not vids:
|
||||
continue
|
||||
pl = Playlist(user_id=demo.id, name=name, position=pos)
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for ipos, vid in enumerate(vids):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=ipos))
|
||||
created += 1
|
||||
return created
|
||||
|
||||
|
||||
def _seed_demo_subscriptions(db: Session, demo: User, n: int = 14) -> int:
|
||||
"""Seed the demo with a handful of catalog channels (the ones with the most videos) so its
|
||||
Channel manager isn't empty — gives visitors something to explore and matches the landing-page
|
||||
screenshot. Idempotent: clears the demo's subscriptions first, then re-adds the top N."""
|
||||
db.execute(delete(Subscription).where(Subscription.user_id == demo.id))
|
||||
top = (
|
||||
db.execute(
|
||||
select(Video.channel_id)
|
||||
.where(Video.channel_id.is_not(None))
|
||||
.group_by(Video.channel_id)
|
||||
.order_by(func.count().desc())
|
||||
.limit(n)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for channel_id in top:
|
||||
db.add(Subscription(user_id=demo.id, channel_id=channel_id))
|
||||
return len(top)
|
||||
|
||||
|
||||
def reset_demo_state(db: Session, demo: User) -> int:
|
||||
"""Wipe the demo account's per-user state (watch/save/hide states, playlists, preferences)
|
||||
back to a clean baseline and re-seed a few sample playlists + channel subscriptions. Returns
|
||||
the seeded playlist count. Shared by the admin's manual reset button and the scheduled demo
|
||||
reset (see scheduler)."""
|
||||
db.execute(delete(VideoState).where(VideoState.user_id == demo.id))
|
||||
# Playlist items cascade on playlist delete (ondelete="CASCADE").
|
||||
db.execute(delete(Playlist).where(Playlist.user_id == demo.id))
|
||||
demo.preferences = {"language": "en"}
|
||||
db.commit()
|
||||
seeded = _seed_demo_playlists(db, demo)
|
||||
_seed_demo_subscriptions(db, demo)
|
||||
db.commit()
|
||||
return seeded
|
||||
@@ -1,4 +1,4 @@
|
||||
"""A user's diagnostics level (E22 S5): what they consented to, and for how long.
|
||||
"""A user's diagnostics level: what they consented to, and for how long.
|
||||
|
||||
Three levels, named in the UI Alap / Részletes / Teljes:
|
||||
|
||||
@@ -10,7 +10,6 @@ Three levels, named in the UI Alap / Részletes / Teljes:
|
||||
|
||||
A raised level always lapses after `ELEVATION_HOURS`: the expiry is computed HERE at consent time,
|
||||
never taken from the client, and an expired level reads as INFO without anything having to clear it.
|
||||
The shared demo account cannot raise its level — its log would mix every visitor's session.
|
||||
|
||||
Core layer: imports `models` and `sysconfig`.
|
||||
"""
|
||||
@@ -27,7 +26,7 @@ DiagnosticsLevel = Literal["INFO", "DEBUG", "TRACE"]
|
||||
LEVELS: tuple[str, ...] = get_args(DiagnosticsLevel)
|
||||
BASE: DiagnosticsLevel = "INFO"
|
||||
ELEVATION_HOURS = 24
|
||||
# How long a debug session records at most (E16, app/debugsession.py). Here, beside the other
|
||||
# How long a debug session records at most (app/debugsession.py). Here, beside the other
|
||||
# consent window, because the start dialog's consent text states it and reads it from `state`.
|
||||
SESSION_MINUTES = 60
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""E24 — the read-model behind the Home surface ("Mit nézzek most?").
|
||||
"""The read-model behind the Home surface ("Mit nézzek most?").
|
||||
|
||||
The anti-paralysis answer layer: one hero suggestion, a cross-source resume rail, and the user's own
|
||||
comfort shelf, all served from our own Postgres with no upstream call at serve time (the same rule
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The time budget — E24's replacement for "what mood are you in?".
|
||||
"""The time budget — the replacement for "what mood are you in?".
|
||||
|
||||
`random-gondolatok` 15/D asked for a mood questionnaire before the recommendations. The same file's
|
||||
later BA-interview note (#23) says the mood half is not what makes this work: the friction is
|
||||
|
||||
@@ -46,8 +46,8 @@ def resolve_ref(db: Session, user: User, kind: str, ref: str) -> tuple[str, obje
|
||||
translation belongs here rather than in five call sites.
|
||||
|
||||
The COLUMN half of every answer comes from `PIN_COLUMNS`; only the LOOKUP differs per kind
|
||||
(a direct `db.get`, a rating-key translation, an int parse plus an ownership check). Before the
|
||||
E24 review the branches spelled the column names out again and `PIN_COLUMNS` had no readers at
|
||||
(a direct `db.get`, a rating-key translation, an int parse plus an ownership check). Before a
|
||||
review the branches spelled the column names out again and `PIN_COLUMNS` had no readers at
|
||||
all — a constant that called itself the one vocabulary while nothing spoke it."""
|
||||
col = PIN_COLUMNS.get(kind)
|
||||
if col is None:
|
||||
|
||||
@@ -156,7 +156,7 @@ class YouTubeAdapter(SourceAdapter):
|
||||
|
||||
class FacebookAdapter(SourceAdapter):
|
||||
key = "facebook"
|
||||
# Measured (E11.5, yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||||
# Measured (yt-dlp 2026.07.04): with the user's cookie, some reels answer a plain request
|
||||
# with "Cannot parse data" and parse only when the request looks like Chrome. Impersonation is a
|
||||
# SECOND attempt behind that one error, not the default: the links that already download with the
|
||||
# cookie keep their proven path, and only the ones stuck at the wall go through curl_cffi. Without
|
||||
@@ -166,7 +166,7 @@ class FacebookAdapter(SourceAdapter):
|
||||
_POST_PAGE_MAX_BYTES = 16 * 1024 * 1024
|
||||
|
||||
def resolve_url(self, url: str, ydl_opts: dict) -> str:
|
||||
# Measured (E11.5 S4): given a group post's link, yt-dlp either finds no formats or — worse —
|
||||
# Measured: given a group post's link, yt-dlp either finds no formats or — worse —
|
||||
# downloads ANOTHER video from the page's feed data. The post's own video is the one Video node
|
||||
# in the page scripts that carry the post id; its watch link downloads (plain first, then the
|
||||
# Chrome retry above). Needs the cookie: signed out, the page is the login wall.
|
||||
@@ -176,7 +176,7 @@ class FacebookAdapter(SourceAdapter):
|
||||
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
|
||||
raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES)
|
||||
page = raw.decode("utf-8", "replace")
|
||||
# Measured (E11.5 review): a refused or expired cookie gets the login page (~450 KB, a login
|
||||
# Measured: a refused or expired cookie gets the login page (~450 KB, a login
|
||||
# form, no post id at all), which is not a post without a video — the user's cookie is the cause.
|
||||
if _quoted(post_id) not in page:
|
||||
raise StatedFailure(POST_NOT_SHOWN_MESSAGE)
|
||||
|
||||
@@ -249,8 +249,8 @@ def usable_cookie(
|
||||
|
||||
If (2) fails the asset is the SHARED (unsalted) one — attaching the cookie would poison the
|
||||
global cache with a private-group video, so we attach nothing. An unreadable blob (rotated key →
|
||||
InvalidToken) also yields None; the download then fails honestly on the login wall (S3's
|
||||
`login_required`)."""
|
||||
InvalidToken) also yields None; the download then fails honestly on the login wall (the
|
||||
`login_required` category)."""
|
||||
host_key = host_key_for_url(url)
|
||||
if host_key is None:
|
||||
return None
|
||||
|
||||
+71
-113
@@ -188,6 +188,51 @@ def clip_duration(spec: dict, source_duration: int | None) -> int | None:
|
||||
|
||||
# --- ffmpeg: trim / crop --------------------------------------------------------------------
|
||||
|
||||
# Every libx264 / aac re-encode in this package uses these, so a re-encoded file is always the
|
||||
# phone-safe shape (`playable`): `-pix_fmt yuv420p` pins 8-bit 4:2:0 — without it libx264 keeps a
|
||||
# 10-bit or 4:4:4 source's pixel format and writes H.264 High 10 / High 4:4:4, which browsers decode
|
||||
# only patchily (the one safe H.264 is 8-bit 4:2:0).
|
||||
X264_ARGS = ["-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p"]
|
||||
AAC_ARGS = ["-c:a", "aac", "-b:a", "192k"]
|
||||
# The last step of every video filter chain a libx264 re-encode runs: 4:2:0 halves the chroma, so
|
||||
# libx264 refuses an odd width or height ("width not divisible by 2") — measured on an odd source
|
||||
# that is not 4:2:0 already (a 4:4:4 or RGB screen capture, 1365x767). A crop of a 4:2:0 source is
|
||||
# rounded to its chroma grid by ffmpeg itself; this holds every other path to the encoder to the same
|
||||
# even frame. Rounding down to even costs at most one pixel row/column.
|
||||
EVEN_FRAME = "scale=trunc(iw/2)*2:trunc(ih/2)*2"
|
||||
|
||||
# The demuxers ffmpeg/ffprobe may use to open a library file — single-file containers only. Without
|
||||
# it ffmpeg picks the demuxer from the file's CONTENT and NAME, and an upload's name is the user's:
|
||||
# a playlist-type demuxer (HLS, DASH, concat) opens whatever local paths its entries name, so a
|
||||
# crafted upload could copy another user's clip or a Plex file into its own converted asset. A file
|
||||
# no demuxer here can open fails the probe, and an unprobeable upload is a download-only 'file'.
|
||||
MEDIA_DEMUXERS = (
|
||||
"mov", # mp4, m4a, mov, 3gp
|
||||
"matroska", # mkv, webm
|
||||
"mpegts",
|
||||
"mpeg",
|
||||
"avi",
|
||||
"asf",
|
||||
"flv",
|
||||
"ogg",
|
||||
"mp3",
|
||||
"aac",
|
||||
"ac3",
|
||||
"eac3",
|
||||
"flac",
|
||||
"wav",
|
||||
"w64",
|
||||
"aiff",
|
||||
"caf",
|
||||
"amr",
|
||||
)
|
||||
INPUT_GUARD = ["-format_whitelist", ",".join(MEDIA_DEMUXERS)]
|
||||
# A downloaded thumbnail is the remote site's bytes under the suffix yt-dlp gave it (`.png`, `.webp`,
|
||||
# `.jpg` — `pipeline._THUMB_SUFFIXES`): only the single-image demuxers those open, so a "thumbnail"
|
||||
# that is really a playlist cannot pull a local file into the poster. Anything else fails, and the
|
||||
# poster falls back to a cut frame (`ensure_poster`).
|
||||
THUMB_GUARD = ["-format_whitelist", "png_pipe,jpeg_pipe,webp_pipe"]
|
||||
|
||||
|
||||
def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]:
|
||||
"""ffmpeg command for a trim/crop edit. `-progress pipe:1` streams machine-readable progress.
|
||||
@@ -214,7 +259,7 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
||||
]
|
||||
if start:
|
||||
cmd += ["-ss", f"{start:.3f}"]
|
||||
cmd += ["-i", str(src)]
|
||||
cmd += [*INPUT_GUARD, "-i", str(src)]
|
||||
if end is not None:
|
||||
dur = end - start
|
||||
if dur > 0:
|
||||
@@ -222,21 +267,9 @@ def build_edit_cmd(src: Path, dest: Path, spec: dict, out_ext: str) -> list[str]
|
||||
|
||||
if reencode:
|
||||
# Input -ss + re-encode is frame-accurate (decodes from the prior keyframe, discards up to
|
||||
# the exact cut). A crop adds the filter; an accurate trim re-encodes with no filter.
|
||||
if crop:
|
||||
cmd += ["-vf", f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}"]
|
||||
cmd += [
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"20",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
]
|
||||
# the exact cut). A crop adds the filter; an accurate trim only evens the frame.
|
||||
cropf = f"crop={crop['w']}:{crop['h']}:{crop['x']}:{crop['y']}," if crop else ""
|
||||
cmd += ["-vf", cropf + EVEN_FRAME, *X264_ARGS, *AAC_ARGS]
|
||||
else:
|
||||
cmd += ["-c", "copy", "-avoid_negative_ts", "make_zero"]
|
||||
|
||||
@@ -284,11 +317,14 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
||||
parts, labels = [], []
|
||||
for i, s in enumerate(segs):
|
||||
st, en = s["start_s"], s["end_s"]
|
||||
parts.append(f"[0:v]trim=start={st}:end={en},setpts=PTS-STARTPTS{cropf}[v{i}]")
|
||||
parts.append(
|
||||
f"[0:v]trim=start={st}:end={en},setpts=PTS-STARTPTS{cropf},{EVEN_FRAME}[v{i}]"
|
||||
)
|
||||
parts.append(f"[0:a]atrim=start={st}:end={en},asetpts=PTS-STARTPTS[a{i}]")
|
||||
labels.append(f"[v{i}][a{i}]")
|
||||
parts.append(f"{''.join(labels)}concat=n={len(segs)}:v=1:a=1[v][a]")
|
||||
cmd = head + [
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-filter_complex",
|
||||
@@ -297,26 +333,23 @@ def build_concat_plan(src: Path, dest: Path, spec: dict, out_ext: str, staging:
|
||||
"[v]",
|
||||
"-map",
|
||||
"[a]",
|
||||
"-c:v",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"veryfast",
|
||||
"-crf",
|
||||
"20",
|
||||
"-c:a",
|
||||
"aac",
|
||||
"-b:a",
|
||||
"192k",
|
||||
*X264_ARGS,
|
||||
*AAC_ARGS,
|
||||
]
|
||||
else:
|
||||
listp = staging / "concat.txt"
|
||||
lines = []
|
||||
lines = ["ffconcat version 1.0"]
|
||||
for s in segs:
|
||||
lines.append(f"file '{_concat_quote(src)}'")
|
||||
# ffmpeg hands the list's whitelist (which must name `concat` to read the list) down to
|
||||
# the file it names, so a library file whose bytes are a concat list would be read as one
|
||||
# and open the paths inside it. Each entry's own whitelist replaces the inherited one.
|
||||
lines.append(f"option format_whitelist {INPUT_GUARD[1]}")
|
||||
lines.append(f"inpoint {s['start_s']}")
|
||||
lines.append(f"outpoint {s['end_s']}")
|
||||
prep[listp] = "\n".join(lines) + "\n"
|
||||
cmd = head + ["-f", "concat", "-safe", "0", "-i", str(listp), "-c", "copy"]
|
||||
guard = ["-format_whitelist", "concat," + INPUT_GUARD[1]]
|
||||
cmd = head + ["-f", "concat", "-safe", "0", *guard, "-i", str(listp), "-c", "copy"]
|
||||
|
||||
if out_ext in ("mp4", "m4a", "mov"):
|
||||
cmd += ["-movflags", "+faststart"]
|
||||
@@ -474,14 +507,7 @@ def run_ffmpeg(
|
||||
raise StatedFailure(f"ffmpeg failed ({ret}): {''.join(err_tail)[-300:]}")
|
||||
|
||||
|
||||
# --- ffmpeg: universal browser playability --------------------------------------------------
|
||||
# A downloaded file must play in EVERY browser, whatever the source shipped. Universal = H.264
|
||||
# video + AAC/MP3 audio in an mp4 container: H.264 is the only video codec all browsers decode
|
||||
# (VP9/AV1 fail on Safari/iOS; HEVC fails on Chrome/Firefox), and mp4/AAC is the safe pairing.
|
||||
# Anything else is fixed post-download: re-encode the offending stream(s), or just remux when the
|
||||
# codecs are already fine but the container isn't.
|
||||
_BROWSER_SAFE_VCODEC = ("h264",)
|
||||
_BROWSER_SAFE_ACODEC = ("aac", "mp3")
|
||||
# --- ffprobe ---------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_ffprobe_json(
|
||||
@@ -493,10 +519,10 @@ def run_ffprobe_json(
|
||||
) -> dict | None:
|
||||
"""One ffprobe call → its parsed JSON dict, or None on any failure (timeout, non-zero exit,
|
||||
missing binary, unparseable output). The SINGLE ffprobe-JSON-invocation point in the downloads
|
||||
package: the browser-playability codec probe here and the shared media probe (`probe.probe_media`,
|
||||
used by both the download worker's finalize and the upload ingest) both go through it, so the
|
||||
package: the shared media probe (`probe.probe_json`: the stored facts and the phone-safe rule) goes
|
||||
through it, so the
|
||||
subprocess flags/timeout/error handling live in one place instead of drifting copies."""
|
||||
args = ["ffprobe", "-v", "error"]
|
||||
args = ["ffprobe", "-v", "error", *INPUT_GUARD]
|
||||
if show_streams:
|
||||
args.append("-show_streams")
|
||||
if show_format:
|
||||
@@ -517,35 +543,6 @@ def run_ffprobe_json(
|
||||
return None
|
||||
|
||||
|
||||
def _codecs_from_streams(streams: list) -> tuple[str | None, str | None]:
|
||||
"""(video_codec, audio_codec) lowercased from an ffprobe JSON stream list — the first of each kind.
|
||||
|
||||
Parsed from the JSON `streams` list, NOT ffprobe's flat `default=nw=1` output: there the fields
|
||||
arrive in ffprobe's own struct order, which puts `codec_name` BEFORE `codec_type`, so a line-by-line
|
||||
"remember the last codec_type" parser attributes every name to the PREVIOUS stream. Measured on real
|
||||
downloads: an ordinary H.264+AAC mp4 probed as `v=aac a=None` — the browser-playability check then
|
||||
believed the video stream was AAC and re-encoded a perfectly fine file on every download. Reading each
|
||||
stream dict by its own `codec_type` is immune to that ordering."""
|
||||
vcodec = acodec = None
|
||||
for s in streams:
|
||||
name = (s.get("codec_name") or "").lower() or None
|
||||
ctype = (s.get("codec_type") or "").lower()
|
||||
if ctype == "video" and vcodec is None:
|
||||
vcodec = name
|
||||
elif ctype == "audio" and acodec is None:
|
||||
acodec = name
|
||||
return vcodec, acodec
|
||||
|
||||
|
||||
def probe_codecs(path: Path) -> tuple[str | None, str | None]:
|
||||
"""Return (video_codec, audio_codec) lowercased via one ffprobe call; None for a missing
|
||||
stream or if ffprobe fails (a failed probe is treated as 'leave it alone' upstream)."""
|
||||
data = run_ffprobe_json(path, entries="stream=codec_type,codec_name")
|
||||
if data is None:
|
||||
return None, None
|
||||
return _codecs_from_streams(data.get("streams") or [])
|
||||
|
||||
|
||||
def probe_duration(path: Path) -> int:
|
||||
"""Media duration in whole seconds via ffprobe, or 0 if unknown — a progress denominator for a
|
||||
re-encode when the yt-dlp info dict carried no `duration` (some URL/live extractions)."""
|
||||
@@ -555,6 +552,7 @@ def probe_duration(path: Path) -> int:
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
*INPUT_GUARD,
|
||||
"-show_entries",
|
||||
"format=duration",
|
||||
"-of",
|
||||
@@ -577,49 +575,6 @@ def probe_duration(path: Path) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def browser_transcode_flags(
|
||||
vcodec: str | None, acodec: str | None, is_mp4: bool
|
||||
) -> tuple[bool, bool, bool]:
|
||||
"""Decide what (if anything) makes a file universally browser-playable. Returns
|
||||
(needs_change, reencode_video, reencode_audio). needs_change=False => already H.264 + AAC/MP3
|
||||
in mp4. A None codec (no such stream, or a failed probe) is never re-encoded."""
|
||||
rv = bool(vcodec) and vcodec not in _BROWSER_SAFE_VCODEC
|
||||
ra = bool(acodec) and acodec not in _BROWSER_SAFE_ACODEC
|
||||
return (rv or ra or not is_mp4), rv, ra
|
||||
|
||||
|
||||
def build_browser_safe_cmd(
|
||||
src: Path, dest: Path, reencode_video: bool, reencode_audio: bool, threads: int | None = None
|
||||
) -> list[str]:
|
||||
"""ffmpeg command that lands `src` as a browser-safe mp4 (faststart): re-encode only the
|
||||
stream(s) that need it (libx264 / aac), stream-copy the rest, always into mp4. `-progress
|
||||
pipe:1` streams machine-readable progress for run_ffmpeg. `threads` caps the encoder so a
|
||||
re-encode can't claim every core on the shared box (see the worker-child de-prioritization)."""
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-nostats",
|
||||
"-i",
|
||||
str(src),
|
||||
]
|
||||
cmd += (
|
||||
["-c:v", "libx264", "-preset", "veryfast", "-crf", "20"]
|
||||
if reencode_video
|
||||
else ["-c:v", "copy"]
|
||||
)
|
||||
cmd += ["-c:a", "aac", "-b:a", "192k"] if reencode_audio else ["-c:a", "copy"]
|
||||
if threads and threads > 0:
|
||||
cmd += ["-threads", str(threads)]
|
||||
cmd += ["-movflags", "+faststart", str(dest)]
|
||||
return cmd
|
||||
|
||||
|
||||
# --- filmstrip (editor scrub track) ---------------------------------------------------------
|
||||
|
||||
_SB_COLS = 12
|
||||
@@ -655,6 +610,7 @@ def build_storyboard_cmd(src: Path, dest: Path, duration_s: int | None, geom: di
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
@@ -678,6 +634,7 @@ def build_poster_cmd(src: Path, dest: Path, at_s: float) -> list[str]:
|
||||
"-nostdin",
|
||||
"-ss",
|
||||
f"{max(0.0, at_s):.3f}",
|
||||
*INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
@@ -699,6 +656,7 @@ def build_thumbnail_jpg_cmd(src: Path, dest: Path) -> list[str]:
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
*THUMB_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
"-frames:v",
|
||||
@@ -734,7 +692,7 @@ def _step_failure(err: BaseException) -> str:
|
||||
|
||||
|
||||
def _is_jpeg(path: Path) -> bool:
|
||||
"""True if the file starts with the JPEG SOI marker (FF D8 FF)."""
|
||||
"""True if the file starts with the JPEG SOI marker (0xFF 0xD8 0xFF)."""
|
||||
try:
|
||||
with path.open("rb") as f:
|
||||
return f.read(3) == b"\xff\xd8\xff"
|
||||
|
||||
@@ -8,7 +8,7 @@ explanation (and decides whether a retry makes sense). The raw text stays availa
|
||||
"details" disclosure. This is the same string-matching approach the worker already trusts for
|
||||
`_RETRIABLE_MARKERS` — kept here as the single source of truth for "what kind of failure is this".
|
||||
|
||||
**The failure catalog** (`_CATALOG`, E11.5). A category is generic: "Sign-in required" cannot tell a
|
||||
**The failure catalog** (`_CATALOG`). A category is generic: "Sign-in required" cannot tell a
|
||||
Facebook user that the cookie they already gave was refused, and yt-dlp words several different
|
||||
Facebook walls identically (`Cannot parse data`). A catalog entry is scoped to ONE host and may also
|
||||
depend on whether a cookie was attached and on the kind of link, and names a `detail` code the UI has
|
||||
@@ -49,10 +49,11 @@ from app.downloads.credentials import host_key_for_url
|
||||
# "produced no output file" have markers here; the urlguard refusal deliberately has none and
|
||||
# classifies as `unknown` — it is a refusal, not a diagnosis, and no category would say more.
|
||||
#
|
||||
# A source adapter's verdicts on a link (`SourceAdapter.resolve_url`, E11.5 S4) are adapters.py's
|
||||
# A source adapter's verdicts on a link (`SourceAdapter.resolve_url`) are adapters.py's
|
||||
# constants, imported above; each has a rule below that gives its category back, and a catalog entry
|
||||
# that gives the site's sentence.
|
||||
QUOTA_FULL_MESSAGE = "Your download storage is full — free up space, then try again."
|
||||
UPLOAD_MISSING_MESSAGE = "The uploaded file is missing."
|
||||
NETWORK_REDACTED_MESSAGE = "The server could not reach that address."
|
||||
|
||||
# (marker, category) — first match wins, so order most-specific → most-general. Matching is
|
||||
@@ -98,6 +99,8 @@ _RULES: tuple[tuple[str, str], ...] = (
|
||||
# An edit's source download went away — our own wording, and MORE specific than the generic
|
||||
# "no longer available" below, so it must be matched first.
|
||||
("source download is no longer available", "source_missing"),
|
||||
# An upload's original went away before its conversion ran — our wording (UPLOAD_MISSING_MESSAGE).
|
||||
(UPLOAD_MISSING_MESSAGE.lower().rstrip("."), "upload_missing"),
|
||||
("source file is missing", "source_missing"),
|
||||
# Private / deleted / removed / terminated.
|
||||
("private video", "unavailable"),
|
||||
@@ -263,6 +266,15 @@ def classify(msg: str | None) -> str:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def classify_conversion(msg: str | None) -> str:
|
||||
"""The category of an upload's failed conversion. The generic ffmpeg categories speak of a
|
||||
download ("the video downloaded but…"), which an upload never was: a failed encode or an
|
||||
unrecognised failure of one is `convert_failed`; a category of its own (a full quota, a missing
|
||||
original) stays as it is."""
|
||||
code = classify(msg)
|
||||
return "convert_failed" if code in ("postprocess", "unknown") else code
|
||||
|
||||
|
||||
def diagnose(
|
||||
msg: str | None, *, url: str | None = None, cookie_attached: bool = False
|
||||
) -> Diagnosis:
|
||||
|
||||
@@ -13,7 +13,7 @@ spec shape (all keys optional; defaults + browser-playability clamp applied by `
|
||||
vcodec always coerced to null → H.264 selected (see `normalize`); no opt-out
|
||||
embed_subs bool embed_chapters bool embed_thumbnail bool sponsorblock bool
|
||||
|
||||
`normalize` clamps container/vcodec/audio_format so EVERY download plays in-browser (E7); the
|
||||
`normalize` clamps container/vcodec/audio_format so EVERY download plays in-browser; the
|
||||
`container`/`vcodec` keys survive only so an old stored profile deserializes without error.
|
||||
"""
|
||||
|
||||
@@ -182,7 +182,9 @@ _DEFAULTS = {
|
||||
# v4: EVERY download is clamped to the mp4/H.264/AAC compat profile (no more mkv/webm/VP9/AV1/Opus
|
||||
# opt-out), so any asset cached under an old non-compat spec — or a stale non-H.264 mp4 that slipped
|
||||
# the v3 net — must re-derive through the current browser-safe guarantee.
|
||||
_SIG_VERSION = 4
|
||||
# v5: the phone-safe rule (`playable`) also judges the pixel format, so a cached 10-bit / 4:4:4 H.264
|
||||
# asset (which v4 let through as "H.264") re-derives as 8-bit 4:2:0 instead of being reused.
|
||||
_SIG_VERSION = 5
|
||||
# Audio-only downloads version on their own, so a change to the audio bytes doesn't re-download every
|
||||
# cached video. v5: an extracted .mp3/.m4a is stored as itself instead of remuxed into an .mp4 that
|
||||
# iPad's WebKit refuses to play, so a cached remuxed audio asset must re-derive.
|
||||
@@ -217,14 +219,15 @@ def normalize(spec: dict) -> dict:
|
||||
# Audio-only files can't embed subtitles/thumbnails as video; keep the flags meaningful.
|
||||
if s["mode"] == "a":
|
||||
s["embed_subs"] = False
|
||||
# Browser-playability guarantee (E7): the Download module exists to download → share → PLAY
|
||||
# Browser-playability guarantee: the Download module exists to download → share → PLAY
|
||||
# in-browser, so every output must play in EVERY browser, iOS/Safari/WebKit included. The only
|
||||
# universally-safe combo is mp4 + H.264 + AAC/MP3, so we CLAMP the spec to it HERE — the single
|
||||
# point every path (picker, stored profile, raw API) flows through — instead of only hiding
|
||||
# options in the UI. Non-mp4 containers (mkv doesn't play at all; webm/VP9/AV1 fail iOS) and any
|
||||
# explicit vcodec are coerced to the compat profile; Opus audio (Safari-flaky as a bare file)
|
||||
# falls back to AAC/m4a. A source that exposes no H.264 is still caught by the worker's
|
||||
# browser-safe re-encode (`_ensure_browser_playable`), so playback is guaranteed end-to-end.
|
||||
# universally-safe combo is mp4 + H.264 + AAC (an .mp3/.m4a for audio), so we CLAMP the spec to
|
||||
# it HERE — the single point every path (picker, stored profile, raw API) flows through — instead
|
||||
# of only hiding options in the UI. Non-mp4 containers (mkv plays nowhere on iOS; AV1 fails there,
|
||||
# and webm/VP9 plays only on recent devices) and any explicit vcodec are coerced to the compat
|
||||
# profile; Opus audio falls back to AAC/m4a. A source that exposes no H.264 (or a 10-bit one) is
|
||||
# still caught by the worker's phone-safe step (`downloads.playable`), so playback is guaranteed
|
||||
# end-to-end.
|
||||
s["container"] = "mp4"
|
||||
s["vcodec"] = None
|
||||
if s["audio_format"] not in ("m4a", "mp3"):
|
||||
@@ -274,7 +277,7 @@ def build_ydl_opts(
|
||||
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
||||
generic options below. Everything here is source-agnostic.
|
||||
|
||||
`cookiefile` is the acting user's per-host cookies.txt path (S6), attached ONLY by the worker
|
||||
`cookiefile` is the acting user's per-host cookies.txt path, attached ONLY by the worker
|
||||
after the host-scope + private-asset gate in `downloads.credentials`. It is set AFTER the `extra`
|
||||
merge so a source adapter can never clobber it.
|
||||
|
||||
@@ -329,7 +332,7 @@ def build_ydl_opts(
|
||||
if max_bytes and max_bytes > 0:
|
||||
opts["max_filesize"] = max_bytes
|
||||
|
||||
# The acting user's per-host cookie (S6). AFTER the adapter merge on purpose: an adapter must not
|
||||
# The acting user's per-host cookie. AFTER the adapter merge on purpose: an adapter must not
|
||||
# be able to override the authenticated session the worker resolved for this specific user.
|
||||
if cookiefile:
|
||||
opts["cookiefile"] = cookiefile
|
||||
@@ -351,7 +354,7 @@ def build_ydl_opts(
|
||||
# inside mp4 — a VP9/AV1-in-mp4 file shows a broken player there — so H.264 is selected at the
|
||||
# SELECTION level, not merely preferred: a sort key can't conjure an H.264 stream a source
|
||||
# doesn't expose as video-only. A source that ships only non-H.264 is caught after download by
|
||||
# the worker's browser-safe re-encode (`_ensure_browser_playable`).
|
||||
# the worker's phone-safe step (`downloads.playable`).
|
||||
if s["mode"] == "v":
|
||||
# Video-only: keep it audio-free. Prefer an H.264 video-only stream, then any video-only,
|
||||
# and only fall to a progressive (audio-bearing) `best` as a last resort.
|
||||
|
||||
+48
-17
@@ -6,12 +6,13 @@ Five passes each run:
|
||||
2. Expiry deletion — delete assets past their TTL (files + row). The FK SET NULL nulls the
|
||||
holders' jobs' asset_id, so those jobs surface as "expired" (re-downloadable) in the UI.
|
||||
3. LRU eviction — if a total cache cap is set and exceeded, delete least-recently-accessed
|
||||
ready assets until back under it.
|
||||
ready assets (never an upload) until back under it.
|
||||
4. Orphan reap — delete errored, fileless, unreferenced asset rows (they carry no TTL, so
|
||||
passes 1-3 skip them; benign row-count hygiene).
|
||||
5. Orphan FILE sweep — delete on-disk files/dirs no surviving asset owns (cruft left when a
|
||||
re-download renames an asset, e.g. old accented channel dirs), skipping system trees and
|
||||
anything newer than an hour so an in-flight download is never touched.
|
||||
re-download renames an asset, e.g. old accented channel dirs, or an upload's stranded file).
|
||||
It skips the dot-prefixed system trees (all but `.uploads`) and anything newer than an hour,
|
||||
so an in-flight download is never touched.
|
||||
|
||||
Runs in the API process (which mounts DOWNLOAD_ROOT). Pure disk/DB work — no YouTube quota.
|
||||
"""
|
||||
@@ -20,19 +21,19 @@ import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.config import settings
|
||||
from app.downloads import storage
|
||||
from app.downloads import quota, storage
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
# Dot-prefixed top-level trees that are NOT part of the Plex/flat asset layout — the orphan sweep
|
||||
# never touches them (user clips, Plex HLS/img caches, in-progress staging, editor storyboards).
|
||||
_SYSTEM_TREE = {".edits", ".plex-hls", ".plex-img-cache", ".staging", ".storyboards"}
|
||||
# The one dot-prefixed top-level tree the orphan sweep enters. The others are not part of the asset
|
||||
# layout (user clips, Plex HLS/img caches, in-progress staging, editor storyboards) and are skipped.
|
||||
_UPLOAD_TREE = ".uploads"
|
||||
_ORPHAN_MIN_AGE_S = 3600 # only sweep files older than this, so an in-flight download is never hit
|
||||
|
||||
|
||||
@@ -67,16 +68,20 @@ def _delete_asset(db: Session, asset: MediaAsset, ntype: str) -> None:
|
||||
def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
"""Delete media/sidecar files (and now-empty dirs) under DOWNLOAD_ROOT that no `ready` asset
|
||||
owns — cruft left when a re-download changes an asset's on-disk name (e.g. an old accented
|
||||
channel dir, or a poster from a superseded naming scheme). Skips the dot-prefixed system trees
|
||||
and only removes files older than `_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download
|
||||
is never deleted, and only files NOT owned by a surviving asset."""
|
||||
channel dir, or a poster from a superseded naming scheme) — or, under `.uploads`, that no upload
|
||||
row names at all. Skips the other dot-prefixed system trees and only removes files older than
|
||||
`_ORPHAN_MIN_AGE_S`, so a just-completed or in-flight download is never deleted, and only files
|
||||
NOT owned by a surviving asset."""
|
||||
root = Path(download_root)
|
||||
if not root.is_dir():
|
||||
return 0
|
||||
keep: set[Path] = set()
|
||||
# An upload's file is on disk from ingest on, whatever its state (a queued conversion's original,
|
||||
# a failed one kept for Retry), so every upload row keeps its file — not only a ready one.
|
||||
for rel in db.execute(
|
||||
select(MediaAsset.rel_path).where(
|
||||
MediaAsset.status == "ready", MediaAsset.rel_path.is_not(None)
|
||||
or_(MediaAsset.status == "ready", MediaAsset.source_kind == "upload"),
|
||||
MediaAsset.rel_path.is_not(None),
|
||||
)
|
||||
).scalars():
|
||||
media = (root / rel).resolve()
|
||||
@@ -99,7 +104,10 @@ def _sweep_orphan_files(db: Session, download_root: str) -> int:
|
||||
|
||||
unreclaimed = 0
|
||||
for entry in root.iterdir():
|
||||
if entry.name.startswith("."): # .edits / .plex-* / .staging / .storyboards + any dotfile
|
||||
# .edits / .plex-* / .staging / .storyboards + any dotfile — but `.uploads` is swept: a
|
||||
# conversion stopped between placing the new file and the commit that points at it (or the
|
||||
# unlink of the original after it) leaves a file no row names, and nothing else finds it.
|
||||
if entry.name.startswith(".") and entry.name != _UPLOAD_TREE:
|
||||
continue
|
||||
files = [entry] if entry.is_file() else [p for p in entry.rglob("*") if p.is_file()]
|
||||
for f in files:
|
||||
@@ -188,17 +196,17 @@ def run_download_gc(db: Session) -> dict:
|
||||
if cap > 0:
|
||||
total = (
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.status == "ready"
|
||||
)
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
if total > cap:
|
||||
# An upload counts toward the total (it is disk like any other) but is never evicted:
|
||||
# it has no source to fetch again, and it lives until its owner deletes it.
|
||||
candidates = (
|
||||
db.execute(
|
||||
select(MediaAsset)
|
||||
.where(MediaAsset.status == "ready")
|
||||
.where(MediaAsset.status == "ready", MediaAsset.source_kind != "upload")
|
||||
.order_by(MediaAsset.last_access_at.asc().nulls_first())
|
||||
)
|
||||
.scalars()
|
||||
@@ -232,6 +240,29 @@ def run_download_gc(db: Session) -> dict:
|
||||
if orphans:
|
||||
db.commit()
|
||||
|
||||
# 4b. Reap unheld uploads. An upload carries no expires_at, is never evicted and never re-enqueued,
|
||||
# so the passes above never free one that lost its last holder outside `release_asset` — a
|
||||
# conversion that finished after its job was cancelled, or one cancelled mid-run (the asset
|
||||
# was `downloading` then). Its file, ready or still the unconverted original, is at `rel_path`.
|
||||
unheld = (
|
||||
db.execute(
|
||||
select(MediaAsset).where(
|
||||
MediaAsset.source_kind == "upload",
|
||||
MediaAsset.ref_count == 0,
|
||||
MediaAsset.status != "downloading",
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for asset in unheld:
|
||||
if asset.rel_path: # no holder is left to tell, so no notice — just the file and the row
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
reaped += 1
|
||||
if unheld:
|
||||
db.commit()
|
||||
|
||||
# 5. Orphan FILE sweep — remove on-disk cruft no surviving asset owns (old accented dirs, stale
|
||||
# posters). Runs last so the keep-set reflects the assets that survived passes 2-4.
|
||||
swept = _sweep_orphan_files(db, settings.download_root)
|
||||
|
||||
@@ -147,13 +147,14 @@ def _watch_tags(token: str, db: Session) -> str | None:
|
||||
return "\n ".join(tags)
|
||||
|
||||
|
||||
def render_watch_html(token: str, index_html: Path, db: Session) -> str | None:
|
||||
"""Return index.html with the OG block replaced for this watch token, or None to serve the
|
||||
generic page unchanged (invalid/expired/not-ready link, or a template without the markers)."""
|
||||
def render_watch_html(token: str, page_html: Path, db: Session) -> str | None:
|
||||
"""Return the watch page (watch.html) with the OG block replaced for this token, or None to
|
||||
serve the generic page unchanged (invalid/expired/not-ready link, or a template without the
|
||||
markers)."""
|
||||
tags = _watch_tags(token, db)
|
||||
if tags is None:
|
||||
return None
|
||||
text = index_html.read_text(encoding="utf-8")
|
||||
text = page_html.read_text(encoding="utf-8")
|
||||
if "<!--OG:START-->" not in text:
|
||||
return None
|
||||
# A function replacement avoids re.sub interpreting backslashes/group refs in `tags`.
|
||||
|
||||
@@ -13,7 +13,8 @@ from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import adapters, credentials, formats, mediakind, probe, quota, service, storage
|
||||
from app.downloads import adapters, credentials, formats, mediakind, playable, probe, quota
|
||||
from app.downloads import service, storage
|
||||
from app.downloads import urlguard
|
||||
from app.downloads.jobdb import (
|
||||
_edit_staging_dir,
|
||||
@@ -58,6 +59,10 @@ _DB_WRITE_INTERVAL_S = 1.0
|
||||
_CANCEL_POLL_INTERVAL_S = 1.0
|
||||
|
||||
_THUMB_SUFFIXES = (".jpg", ".jpeg", ".png", ".webp")
|
||||
# The job phase while a file is brought into the phone-safe shape (`_ensure_phone_safe`) for a
|
||||
# download or an upload. Its own word, not yt-dlp's "processing": it reports ffmpeg's real progress,
|
||||
# so the queue row shows a % for it, where yt-dlp's post-steps only get an indeterminate pulse.
|
||||
CONVERTING = "converting"
|
||||
# yt-dlp's working files in the staging dir. None of them is ever the finished media, but several
|
||||
# CAN be the biggest file there (a video-only per-format stream, an abandoned fragment run), so the
|
||||
# staging fallback must exclude them by name — picking one would store e.g. a silent video-only
|
||||
@@ -73,9 +78,9 @@ _YTDLP_TEMP_RE = re.compile(
|
||||
|
||||
def _notify_download_done(db, user_id: int | None, title: str | None, job_id, *, kind: str) -> None:
|
||||
"""Post a best-effort 'ready' notice to the owner's inbox — downloads/edits used to finish
|
||||
silently while scheduler jobs and playlist-push already notified (R11 feedback symmetry). A
|
||||
silently while scheduler jobs and playlist-push already notified (feedback symmetry). A
|
||||
notification failure must never mask the completed job, so it is isolated + rolled back (mirrors
|
||||
scheduler._notify_done). `kind` ("download"/"edit") lets the inbox pick accurate copy; the
|
||||
scheduler._notify_done). `kind` ("download"/"edit"/"upload") lets the inbox pick accurate copy; the
|
||||
English title here is only a fallback for an unknown type. `user_id`/`title` are captured by the
|
||||
caller BEFORE its commit, so reading them here needs no post-commit refresh of an expired row."""
|
||||
if user_id is None:
|
||||
@@ -219,7 +224,7 @@ def _ffmpeg_progress_writer(job_id: int, phase: str, *, now=time.monotonic):
|
||||
when the whole-percent value changed (the bar shows integers), plus a separate once-per-interval
|
||||
heartbeat while the percent is stuck (so a slow re-encode isn't mistaken for a stall)."""
|
||||
# Progress writes have their OWN 1/interval budget so a heartbeat can never delay a real write (the
|
||||
# round-1 concern). The heartbeat, in turn, is gated on `last_touch` — the last DB write OR beat —
|
||||
# original concern). The heartbeat, in turn, is gated on `last_touch` — the last DB write OR beat —
|
||||
# so a progress write (which already refreshes updated_at) suppresses a redundant heartbeat that
|
||||
# same interval, keeping active downloads at ~1 write/interval, not two.
|
||||
allow_write = _rate_limiter(_DB_WRITE_INTERVAL_S, now=now)
|
||||
@@ -341,7 +346,7 @@ def _extract_with_fallback(
|
||||
(adapters.POST_*_MESSAGE) instead of letting yt-dlp guess, and a failed page read fails the
|
||||
download as itself.
|
||||
|
||||
`cookiefile` (S6) is the acting user's per-host cookies.txt, already gated by
|
||||
`cookiefile` is the acting user's per-host cookies.txt, already gated by
|
||||
`credentials.resolve_cookiefile`; passed to every attempt so the fallback stays authenticated.
|
||||
|
||||
`max_bytes` (SEC-13) is the owner's remaining storage budget, applied to every attempt."""
|
||||
@@ -416,48 +421,50 @@ def _download_settings() -> tuple[str, int]:
|
||||
)
|
||||
|
||||
|
||||
def _ensure_browser_playable(job_id: int, produced: Path, spec: dict, info: dict) -> Path:
|
||||
"""Make an mp4-compat-profile download play in every browser: H.264 + AAC/MP3 in mp4. If the
|
||||
download (whatever the source shipped) isn't already that, re-encode the offending stream(s) —
|
||||
or just remux when the codecs are fine but the container isn't. **Scope:** only the mp4 compat
|
||||
profile is normalized; an explicit `vcodec` or a non-mp4 custom profile is the user opting out
|
||||
of the guarantee and is left as-is (so a `webm` profile can still ship VP9/Opus). Returns the
|
||||
file to store (converted, or the original) and corrects `info`'s codec fields to match disk.
|
||||
def _ensure_phone_safe(
|
||||
job_id: int,
|
||||
produced: Path,
|
||||
out_dir: Path,
|
||||
phase: str,
|
||||
total_s: int | None = None,
|
||||
*,
|
||||
keep_source: bool = False,
|
||||
) -> Path:
|
||||
"""Bring a file that is about to become a ready library asset into the phone-safe shape
|
||||
(`playable`), writing any converted copy into `out_dir`. Returns the file to store — `produced`
|
||||
itself when it already is that shape, or isn't media at all — and removes `produced` once a
|
||||
converted copy exists, unless `keep_source` (an upload's original, which must outlive the
|
||||
conversion until the database points at the new file). The job shows `phase` with real ffmpeg
|
||||
progress while it converts.
|
||||
|
||||
An audio download (mode "a") is never touched: yt-dlp already extracted it to .mp3 / .m4a, which
|
||||
every browser plays as audio/mpeg / audio/mp4. Remuxing it into .mp4 served an mp3 as video/mp4,
|
||||
which iPad's WebKit refuses to play (prod job 268)."""
|
||||
if spec.get("mode") == "a" or spec.get("container") != "mp4" or spec.get("vcodec"):
|
||||
Every ready path calls this — a download, an upload's worker conversion and an editor clip — so
|
||||
a share link can only ever serve the one shape every phone and browser plays."""
|
||||
p = playable.plan(probe.probe_json(produced), produced.suffix)
|
||||
if p is None or not p.needs_change:
|
||||
return produced
|
||||
vcodec, acodec = editmod.probe_codecs(produced)
|
||||
is_mp4 = produced.suffix.lower() == ".mp4"
|
||||
needs, reenc_v, reenc_a = editmod.browser_transcode_flags(vcodec, acodec, is_mp4)
|
||||
if not needs:
|
||||
return produced
|
||||
|
||||
dest = produced.with_name(produced.stem + ".browser.mp4")
|
||||
action = "re-encode" if (reenc_v or reenc_a) else "remux"
|
||||
dest = out_dir / f"{produced.stem}.safe.{p.ext}"
|
||||
action = "re-encode" if (p.reencode_video or p.reencode_audio) else "remux"
|
||||
log.info(
|
||||
"job %s: %s (v=%s a=%s mp4=%s) -> browser-safe mp4", job_id, action, vcodec, acodec, is_mp4
|
||||
"job %s: %s (video=%s audio=%s) -> phone-safe .%s",
|
||||
job_id,
|
||||
action,
|
||||
p.reencode_video,
|
||||
p.reencode_audio,
|
||||
p.ext,
|
||||
extra=logs.event("download.phone_safe", job_id=job_id, action=action, ext=p.ext),
|
||||
)
|
||||
_set_job(job_id, phase="processing", progress=0, speed_bps=None, eta_s=None)
|
||||
# Prefer the info duration; probe the file when the extractor didn't provide one, so a long
|
||||
# re-encode still shows real progress instead of a frozen 0%.
|
||||
total_s = int(info["duration"]) if info.get("duration") else editmod.probe_duration(produced)
|
||||
_set_job(job_id, phase=phase, progress=0, speed_bps=None, eta_s=None)
|
||||
# A re-encode with no known length would hold a frozen 0%; probe the file for a denominator.
|
||||
editmod.run_ffmpeg(
|
||||
editmod.build_browser_safe_cmd(
|
||||
produced, dest, reenc_v, reenc_a, threads=editmod.encode_thread_cap()
|
||||
),
|
||||
total_s=total_s,
|
||||
on_progress=_ffmpeg_progress_writer(job_id, "processing"),
|
||||
playable.build_cmd(produced, dest, p, threads=editmod.encode_thread_cap()),
|
||||
total_s=total_s or editmod.probe_duration(produced),
|
||||
on_progress=_ffmpeg_progress_writer(job_id, phase),
|
||||
should_cancel=_cancel_poll(job_id),
|
||||
)
|
||||
produced.unlink(missing_ok=True)
|
||||
# Keep the stored asset metadata truthful about what's now on disk.
|
||||
if reenc_v:
|
||||
info["vcodec"] = "h264"
|
||||
if reenc_a:
|
||||
info["acodec"] = "aac"
|
||||
if not dest.exists():
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
if not keep_source:
|
||||
produced.unlink(missing_ok=True)
|
||||
return dest
|
||||
|
||||
|
||||
@@ -483,7 +490,7 @@ def _download(
|
||||
urlguard.check_url(url)
|
||||
except urlguard.BlockedUrl as e:
|
||||
raise StatedFailure(f"Refusing to download from this address ({e.reason}).") from e
|
||||
# S6: attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# Attach the acting user's per-host cookie ONLY when it's host-scoped to this URL AND
|
||||
# this asset is that user's private, salted identity (the double gate lives in credentials).
|
||||
remaining_bytes: int | None = None
|
||||
with SessionLocal() as cdb:
|
||||
@@ -522,7 +529,13 @@ def _download(
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||
|
||||
produced = _produced_file(info, staging)
|
||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||
produced = _ensure_phone_safe(
|
||||
job_id,
|
||||
produced,
|
||||
staging,
|
||||
CONVERTING,
|
||||
int(info["duration"]) if info.get("duration") else None,
|
||||
)
|
||||
ext = produced.suffix.lstrip(".").lower()
|
||||
# Codecs/dims/duration come from a PROBE of the produced file — the ground truth — NOT yt-dlp's
|
||||
# `info` dict, whose fields are absent for some sources (Facebook among them): deriving
|
||||
@@ -722,9 +735,12 @@ def _log_download_failed(job_id: int, msg: str, code: str | None, exc: BaseExcep
|
||||
)
|
||||
|
||||
|
||||
def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None = None) -> None:
|
||||
"""Fail an edit job, and say so ONCE — every way an edit fails comes through here, so this is
|
||||
where its line and its event code live. `exc` is the exception when there was one.
|
||||
def _fail_edit(
|
||||
job_id: int, asset_id: int, msg: str, exc: BaseException | None = None, *, what: str = "edit"
|
||||
) -> None:
|
||||
"""Fail a local ffmpeg job — an edit, or an upload's conversion (`what="upload"`) — and say so
|
||||
ONCE: every way one fails comes through here, so this is where its line and its event code
|
||||
(`<what>.job.failed`) live. `exc` is the exception when there was one.
|
||||
|
||||
Its traceback goes with the line unless the message already tells it all (`_failure_stack`).
|
||||
Logged after the write, as on the download error path."""
|
||||
@@ -739,19 +755,22 @@ def _fail_edit(job_id: int, asset_id: int, msg: str, exc: BaseException | None =
|
||||
job_id,
|
||||
status=JobStatus.ERROR,
|
||||
error=msg,
|
||||
error_code=dlerrors.classify(msg),
|
||||
error_code=dlerrors.classify_conversion(msg)
|
||||
if what == "upload"
|
||||
else dlerrors.classify(msg),
|
||||
phase=None,
|
||||
)
|
||||
db.commit()
|
||||
# `msg` is the user's copy; ffmpeg's stderr tail in it can name the source file, whose path is
|
||||
# built from the channel and the title.
|
||||
log.warning(
|
||||
"edit job %s failed: %s",
|
||||
"%s job %s failed: %s",
|
||||
what,
|
||||
job_id,
|
||||
formats.log_safe(msg),
|
||||
extra={
|
||||
**logs.event(
|
||||
"edit.job.failed",
|
||||
f"{what}.job.failed",
|
||||
job_id=job_id,
|
||||
asset_id=asset_id,
|
||||
**({"error_type": type(exc).__name__} if exc is not None else {}),
|
||||
@@ -811,11 +830,17 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
)
|
||||
if not dest_staging.exists():
|
||||
raise StatedFailure("ffmpeg produced no output file")
|
||||
# A fast trim stream-copies the source's codecs and container, so the clip is only as
|
||||
# playable as its source: hold it to the same rule as every other ready file.
|
||||
dest_staging = _ensure_phone_safe(job_id, dest_staging, staging, "editing", total_s)
|
||||
out_ext = dest_staging.suffix.lstrip(".").lower() or out_ext
|
||||
|
||||
rel = storage.edit_rel_path(user_id, asset_id, out_ext)
|
||||
storage.place_file(dest_staging, settings.download_root, rel)
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
size = final.stat().st_size if final.exists() else None
|
||||
# The clip's real frame: a re-encode evens an odd crop or source (`editmod.EVEN_FRAME`).
|
||||
_, _, out_w, out_h, _ = probe.probe_media(final)
|
||||
_, retention_days = _download_settings()
|
||||
expires = datetime.now(timezone.utc) + timedelta(days=retention_days)
|
||||
|
||||
@@ -827,7 +852,9 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
asset.size_bytes = size
|
||||
asset.container = out_ext
|
||||
asset.duration_s = total_s or asset.duration_s
|
||||
if crop:
|
||||
if out_w and out_h:
|
||||
asset.width, asset.height = out_w, out_h
|
||||
elif crop:
|
||||
asset.width = edit_spec["crop"]["w"]
|
||||
asset.height = edit_spec["crop"]["h"]
|
||||
else:
|
||||
@@ -867,3 +894,163 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc)
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
|
||||
def _check_converted_fits(owner_id: int | None, old_size: int, new_size: int) -> None:
|
||||
"""Refuse a converted upload that would push its owner over the storage cap: the upload was
|
||||
charged at its original size, and a re-encode can come out larger. The original stays, counted,
|
||||
for the owner to delete or keep."""
|
||||
if owner_id is None:
|
||||
return
|
||||
with SessionLocal() as db:
|
||||
if not quota.fits_after_conversion(db, owner_id, old_size, new_size):
|
||||
raise StatedFailure(dlerrors.QUOTA_FULL_MESSAGE)
|
||||
|
||||
|
||||
def _unplace(placed: Path | None) -> None:
|
||||
"""Remove a converted upload that was placed at its new rel but never committed: the row still
|
||||
names the original, so nothing else would ever find this file."""
|
||||
if placed is not None:
|
||||
placed.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _conversion_landed(asset_id: int, new_rel: str) -> bool | None:
|
||||
"""After the commit that makes a converted upload ready RAISED: did it land anyway? A commit can
|
||||
reach the server and lose its answer (the connection drops after COMMIT). True/False by what the
|
||||
row now says; None when the row cannot be read either."""
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
return asset is not None and asset.status == "ready" and asset.rel_path == new_rel
|
||||
except Exception: # noqa: BLE001 — "cannot tell" is the answer, the caller acts on it
|
||||
# silent-ok: the caller re-raises the commit's own error, which `_fail_edit` reports.
|
||||
return None
|
||||
|
||||
|
||||
def _drop_original(job_id: int, src: Path) -> None:
|
||||
"""Remove an upload's original once the converted file is committed in its place. Best effort:
|
||||
the job is done and its file is ready, so a failure here must not fail it — the original is
|
||||
named by no row now, and the GC's `.uploads` sweep takes it."""
|
||||
try:
|
||||
src.unlink(missing_ok=True)
|
||||
except OSError as exc:
|
||||
log.warning(
|
||||
"upload job %s: original not removed, left to the GC sweep: %s",
|
||||
job_id,
|
||||
exc,
|
||||
extra=logs.event("upload.original.unlink_failed", job_id=job_id),
|
||||
)
|
||||
|
||||
|
||||
def _free_unheld_upload(asset_id: int) -> bool:
|
||||
"""Free an upload no job holds any more — cancelled or deleted while its conversion ran, when
|
||||
`release_asset` had to leave it because the conversion still read its file. Once that has
|
||||
stopped, free it as the release would have; left alone, its file would sit on disk counted by no
|
||||
owner's footprint until the GC's upload pass. False when a job still holds it (paused)."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset is None or asset.ref_count != 0:
|
||||
return False
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
db.commit()
|
||||
return True
|
||||
|
||||
|
||||
def _convert_upload(job_id: int, asset_id: int) -> None:
|
||||
"""Convert a queued upload into the phone-safe shape (`playable`) and make it ready.
|
||||
|
||||
The original sits at the asset's `rel_path` (see `upload`); the converted file replaces it there
|
||||
under the new extension, and only then is the original removed — so a failed or stopped
|
||||
conversion leaves the original in place for Retry, and a delete still finds it by `rel_path`.
|
||||
A converted file placed but never committed is removed again on the way out. When the commit's
|
||||
own outcome is unknown neither file is removed: the row names one of them, and the GC's `.uploads`
|
||||
sweep takes the one it does not — as it does an original whose unlink after the commit failed."""
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
old_rel = asset.rel_path if asset else None
|
||||
old_ext = asset.container if asset else None
|
||||
old_size = (asset.size_bytes or 0) if asset else 0
|
||||
duration = asset.duration_s if asset else None
|
||||
job = db.get(DownloadJob, job_id)
|
||||
owner_id = job.user_id if job else None
|
||||
src = storage.safe_abs_path(settings.download_root, old_rel) if old_rel else None
|
||||
if old_rel is None or src is None or not src.exists():
|
||||
_fail_edit(job_id, asset_id, dlerrors.UPLOAD_MISSING_MESSAGE, what="upload")
|
||||
return
|
||||
staging = _staging_dir(asset_id)
|
||||
placed: Path | None = None # the converted file while it is at its new rel, uncommitted
|
||||
try:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
safe = _ensure_phone_safe(job_id, src, staging, CONVERTING, duration, keep_source=True)
|
||||
ext = safe.suffix.lstrip(".").lower()
|
||||
new_rel = storage.with_ext(old_rel, old_ext, ext)
|
||||
if safe != src:
|
||||
_check_converted_fits(owner_id, old_size, safe.stat().st_size)
|
||||
storage.place_file(safe, settings.download_root, new_rel)
|
||||
if new_rel != old_rel:
|
||||
placed = storage.abs_path(settings.download_root, new_rel)
|
||||
final = storage.abs_path(settings.download_root, new_rel)
|
||||
vcodec, acodec, width, height, dur = probe.probe_media(final)
|
||||
with SessionLocal() as db:
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if asset:
|
||||
asset.status = "ready"
|
||||
asset.rel_path = new_rel
|
||||
asset.container = ext
|
||||
asset.size_bytes = final.stat().st_size
|
||||
asset.vcodec, asset.acodec = vcodec, acodec
|
||||
asset.width, asset.height = width or asset.width, height or asset.height
|
||||
asset.duration_s = dur or asset.duration_s
|
||||
asset.error = None
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
title = asset.title if asset else None
|
||||
finalized = service.finalize_if_running(
|
||||
db, job_id, status=JobStatus.DONE, progress=100, phase=None
|
||||
)
|
||||
try:
|
||||
db.commit()
|
||||
except Exception:
|
||||
landed = _conversion_landed(asset_id, new_rel)
|
||||
if landed is False:
|
||||
raise # rolled back: the original is still the asset's file — fail as any step
|
||||
# Landed, or cannot tell: the converted file may be the asset's file now, so it is
|
||||
# never removed blind (the sweep takes whichever file no row names).
|
||||
placed = None
|
||||
if landed is None:
|
||||
raise
|
||||
db.rollback()
|
||||
placed = None
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, title, job_id, kind="upload")
|
||||
if new_rel != old_rel:
|
||||
_drop_original(job_id, src)
|
||||
if not finalized:
|
||||
_free_unheld_upload(asset_id) # finished after its job was cancelled or deleted
|
||||
if finalized:
|
||||
log.info(
|
||||
"upload job %s converted: asset %s",
|
||||
job_id,
|
||||
asset_id,
|
||||
extra=logs.event("upload.job.done", job_id=job_id, asset_id=asset_id, ext=ext),
|
||||
)
|
||||
except editmod.EditAborted:
|
||||
# Paused/canceled mid-encode: the original is untouched. A paused job keeps it for its
|
||||
# resume; one cancelled or deleted holds it no more, so it is freed now.
|
||||
_unplace(placed)
|
||||
if not _free_unheld_upload(asset_id):
|
||||
_reset_asset_pending(asset_id)
|
||||
log.info(
|
||||
"upload job %s conversion aborted (pause/cancel)",
|
||||
job_id,
|
||||
extra=logs.event("upload.job.aborted", job_id=job_id),
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — surface the failure, keep the worker alive
|
||||
# silent-ok: not silent — `_fail_edit` is the report, and it takes `exc` for its stack.
|
||||
_unplace(placed)
|
||||
_fail_edit(job_id, asset_id, str(exc)[:500], exc, what="upload")
|
||||
|
||||
finally:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""The phone-safe rule: what a library file must be so it plays wherever a share link lands.
|
||||
|
||||
A shared `/watch` link is opened by whoever got it, on whatever they hold: an iPhone's WebKit, an
|
||||
Android Chrome, a desktop Firefox. The one shape every one of them plays:
|
||||
|
||||
* video → H.264, 8-bit 4:2:0 (`yuv420p`, or its full-range twin `yuvj420p`), AAC audio, in .mp4
|
||||
* audio only → MP3 in .mp3, or AAC in .m4a
|
||||
|
||||
Measured on the owner's iPhone + iPad (2026-10-05): AV1, Matroska, HEVC tagged `hev1` and MP3
|
||||
inside an .mp4 all fail there. VP9/WebM, HEVC `hvc1` and 10-bit H.264 played on those devices, but
|
||||
Chrome and Firefox do not decode HEVC and browsers decode 10-bit / 4:4:4 H.264 only patchily, so the
|
||||
rule is the shape every browser plays, not the shape one device plays.
|
||||
|
||||
Every file that becomes a ready library asset passes through here — a download, an upload (the
|
||||
worker converts it, see `upload`), and an editor clip — so a link can only ever serve this shape.
|
||||
`plan` says what (if anything) must change; `build_cmd` is the one ffmpeg command that changes it:
|
||||
re-encode only the stream that fails, copy the rest, or just remux when only the container is wrong.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import probe
|
||||
|
||||
_SAFE_VCODEC = "h264"
|
||||
_SAFE_PIX_FMTS = frozenset({"yuv420p", "yuvj420p"})
|
||||
_VIDEO_EXT = "mp4"
|
||||
# An audio-only file: the codec, and the extension it plays in. Anything else becomes AAC in .m4a.
|
||||
_AUDIO_HOME = {"mp3": "mp3", "aac": "m4a"}
|
||||
_AUDIO_FALLBACK_EXT = "m4a"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Plan:
|
||||
"""What makes one file phone-safe. `ext` is the extension the safe file has (no dot)."""
|
||||
|
||||
ext: str
|
||||
has_video: bool
|
||||
reencode_video: bool
|
||||
reencode_audio: bool
|
||||
needs_change: bool
|
||||
|
||||
|
||||
def plan(data: dict | None, ext: str) -> Plan | None:
|
||||
"""The phone-safe plan for a file whose `probe.probe_json` is `data` and whose extension is
|
||||
`ext`; None when it is not media at all (an unreadable probe, a still image, no audio or video
|
||||
stream) — such a file is a download-only 'file' and never plays inline, so there is nothing to
|
||||
make safe."""
|
||||
if data is None:
|
||||
return None
|
||||
video, audio = probe.primary_streams(data)
|
||||
ext = ext.lower().lstrip(".")
|
||||
if video is not None:
|
||||
rv = (video.get("codec_name") or "").lower() != _SAFE_VCODEC or (
|
||||
video.get("pix_fmt") or ""
|
||||
).lower() not in _SAFE_PIX_FMTS
|
||||
ra = audio is not None and (audio.get("codec_name") or "").lower() != "aac"
|
||||
return Plan(_VIDEO_EXT, True, rv, ra, rv or ra or ext != _VIDEO_EXT)
|
||||
if audio is not None:
|
||||
home = _AUDIO_HOME.get((audio.get("codec_name") or "").lower())
|
||||
target = home or _AUDIO_FALLBACK_EXT
|
||||
return Plan(target, False, False, home is None, home is None or ext != target)
|
||||
return None
|
||||
|
||||
|
||||
def build_cmd(src: Path, dest: Path, p: Plan, threads: int | None = None) -> list[str]:
|
||||
"""The ffmpeg command that writes `src` to `dest` (whose suffix is `p.ext`) in the phone-safe
|
||||
shape: the failing stream(s) re-encoded with the package's shared encoder settings, the rest
|
||||
stream-copied. An audio-only result drops any cover-art stream. `-progress pipe:1`
|
||||
streams machine-readable progress; `threads` caps the encoder so a re-encode cannot claim every
|
||||
core on the shared box (see the worker-child de-prioritization)."""
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"error",
|
||||
"-nostdin",
|
||||
"-progress",
|
||||
"pipe:1",
|
||||
"-nostats",
|
||||
*editmod.INPUT_GUARD,
|
||||
"-i",
|
||||
str(src),
|
||||
]
|
||||
# Exactly the two streams `plan` judged: `V` is a video stream that is not cover art. Nothing
|
||||
# else rides along — a subtitle or data track the .mp4 muxer cannot take would fail the run.
|
||||
if p.has_video:
|
||||
cmd += ["-map", "0:V:0", "-map", "0:a:0?"]
|
||||
if p.reencode_video:
|
||||
cmd += ["-vf", editmod.EVEN_FRAME, *editmod.X264_ARGS]
|
||||
else:
|
||||
cmd += ["-c:v", "copy"]
|
||||
else:
|
||||
cmd += ["-map", "0:a:0"]
|
||||
cmd += editmod.AAC_ARGS if p.reencode_audio else ["-c:a", "copy"]
|
||||
if threads and threads > 0:
|
||||
cmd += ["-threads", str(threads)]
|
||||
if p.ext in ("mp4", "m4a"):
|
||||
cmd += ["-movflags", "+faststart"]
|
||||
cmd += [str(dest)]
|
||||
return cmd
|
||||
@@ -34,6 +34,28 @@ _IMAGE_FORMATS = {
|
||||
}
|
||||
|
||||
|
||||
def primary_streams(data: dict) -> tuple[dict | None, dict | None]:
|
||||
"""(video stream, audio stream) — the first of each kind — from a NON-None ffprobe JSON dict, by
|
||||
the same reading the stored facts use: a still image / image-sequence container has neither (it
|
||||
is a generic 'file'), and a video stream flagged `attached_pic` (an mp3/m4a's cover art) is not a
|
||||
video track. The phone-safe rule (`playable`) judges exactly these two streams."""
|
||||
fmt_names = {
|
||||
n.strip() for n in ((data.get("format") or {}).get("format_name") or "").split(",")
|
||||
}
|
||||
if fmt_names & _IMAGE_FORMATS:
|
||||
return None, None
|
||||
video = audio = None
|
||||
for stream in data.get("streams", []):
|
||||
kind = stream.get("codec_type")
|
||||
if kind == "video" and video is None:
|
||||
if (stream.get("disposition") or {}).get("attached_pic"):
|
||||
continue # cover art, not a video track
|
||||
video = stream
|
||||
elif kind == "audio" and audio is None:
|
||||
audio = stream
|
||||
return video, audio
|
||||
|
||||
|
||||
def _facts_from_probe(
|
||||
data: dict,
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
@@ -45,19 +67,11 @@ def _facts_from_probe(
|
||||
fmt_names = {n.strip() for n in (fmt.get("format_name") or "").split(",")}
|
||||
if fmt_names & _IMAGE_FORMATS:
|
||||
return None, None, None, None, None
|
||||
|
||||
vcodec = acodec = None
|
||||
width = height = None
|
||||
for stream in data.get("streams", []):
|
||||
kind = stream.get("codec_type")
|
||||
if kind == "video" and vcodec is None:
|
||||
if (stream.get("disposition") or {}).get("attached_pic"):
|
||||
continue # cover art, not a video track
|
||||
vcodec = stream.get("codec_name")
|
||||
width = stream.get("width")
|
||||
height = stream.get("height")
|
||||
elif kind == "audio" and acodec is None:
|
||||
acodec = stream.get("codec_name")
|
||||
video, audio = primary_streams(data)
|
||||
vcodec = video.get("codec_name") if video else None
|
||||
width = video.get("width") if video else None
|
||||
height = video.get("height") if video else None
|
||||
acodec = audio.get("codec_name") if audio else None
|
||||
|
||||
duration = None
|
||||
try:
|
||||
@@ -71,16 +85,29 @@ def _facts_from_probe(
|
||||
return vcodec, acodec, width, height, duration
|
||||
|
||||
|
||||
def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
|
||||
or the file isn't playable media (an unprobeable file — or a still image — is treated as a generic
|
||||
non-media file, so it becomes a download-only 'file' rather than a broken 'video')."""
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
def probe_json(path: Path) -> dict | None:
|
||||
"""The full ffprobe JSON (streams + format) of `path`, or None when ffprobe could not read it —
|
||||
the one call both the stored facts (`facts`) and the phone-safe rule read from."""
|
||||
return editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
|
||||
|
||||
def facts(
|
||||
data: dict | None,
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) of a `probe_json` result; all None when it is None
|
||||
(an unprobeable file is treated as a generic non-media file, so it becomes a download-only 'file'
|
||||
rather than a broken 'video')."""
|
||||
if data is None:
|
||||
return None, None, None, None, None
|
||||
return _facts_from_probe(data)
|
||||
|
||||
|
||||
def probe_media(path: Path) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
"""(vcodec, acodec, width, height, duration_s) from one ffprobe call; all None if ffprobe fails
|
||||
or the file isn't playable media (a still image is a generic 'file', see `primary_streams`)."""
|
||||
return facts(probe_json(path))
|
||||
|
||||
|
||||
def resolve_media_facts(
|
||||
path: Path, info: dict
|
||||
) -> tuple[str | None, str | None, int | None, int | None, int | None]:
|
||||
@@ -94,18 +121,18 @@ def resolve_media_facts(
|
||||
is authoritative — INCLUDING a deliberate None (cover-art-only audio, a still image). A per-field
|
||||
`probe or info` fallback would let `info`'s stale vcodec resurrect a video track the file doesn't
|
||||
have (yt-dlp reports a progressive `vcodec` for an audio-mode download whose video it then strips),
|
||||
mislabelling an audio download as a broken in-app 'video'. Only when ffprobe could NOT read the
|
||||
file at all do we fall back to `info` (the pre-probe behaviour, a safety net); the cosmetic
|
||||
dimensions and the duration still gap-fill from `info` in either case."""
|
||||
mislabelling an audio download as a broken in-app 'video'. When ffprobe could NOT read the file
|
||||
at all, no codec is reported: the probe runs with the media demuxer whitelist, so a file it
|
||||
cannot open is one the player and the editor cannot open either — and taking `info`'s codecs
|
||||
for it would make it an editable 'video' whose bytes are whatever the remote site served. The
|
||||
cosmetic dimensions and the duration still gap-fill from `info` in either case."""
|
||||
data = editmod.run_ffprobe_json(path, show_streams=True, show_format=True)
|
||||
info_v = info.get("vcodec") if info.get("vcodec") not in (None, "none") else None
|
||||
info_a = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
info_w = int(info["width"]) if info.get("width") else None
|
||||
info_h = int(info["height"]) if info.get("height") else None
|
||||
info_d = int(info["duration"]) if info.get("duration") else None
|
||||
if data is None:
|
||||
# ffprobe could not read the file — fall back to the extractor's info entirely.
|
||||
return info_v, info_a, info_w, info_h, info_d
|
||||
# Unreadable to the guarded probe: a download-only 'file' (no codecs → media_kind FILE).
|
||||
return None, None, info_w, info_h, info_d
|
||||
pv, pa, pw, ph, pd = _facts_from_probe(data)
|
||||
# Probe ran → its codecs win outright (a None means "no such track", not "unknown"); info only
|
||||
# gap-fills the width/height/duration the probe couldn't read.
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""Per-user download limits + footprint accounting.
|
||||
|
||||
A user's *footprint* is the total size of the ready assets their library references (cache hits
|
||||
count — it's "how much disk you're responsible for", not "how much you personally downloaded").
|
||||
count — it's "how much disk you're responsible for", not "how much you personally downloaded"). An
|
||||
upload counts from the moment it lands, ready or not: its original is on disk while its conversion
|
||||
waits, and stays there when the conversion fails (for Retry) — bytes a ready-only sum never saw.
|
||||
Limits come from a per-user DownloadQuota row if the admin set one, else the sysconfig defaults.
|
||||
`unlimited` (e.g. the admin's own account) bypasses the byte cap.
|
||||
|
||||
@@ -18,16 +20,24 @@ is unknown until it finishes. Edit jobs do not consult it at all (tracked as SEC
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, DownloadQuota, MediaAsset
|
||||
|
||||
# Jobs a user is considered to be "holding" (occupy a job slot / footprint). Terminal failures
|
||||
# (error, canceled) don't count.
|
||||
_HOLDING = ("queued", "running", "paused", "done")
|
||||
_RUNNING = ("queued", "running")
|
||||
_HOLDING = (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED, JobStatus.DONE)
|
||||
_RUNNING = (JobStatus.QUEUED, JobStatus.RUNNING)
|
||||
# An asset whose file is on disk: a ready one, or an upload in any state (its original lands at
|
||||
# ingest, see the module docstring). Every byte sum — a user's footprint, the GC's total-cap pass,
|
||||
# the admin's storage view — reads it, so they all count the same disk.
|
||||
ON_DISK = or_(
|
||||
MediaAsset.status == "ready",
|
||||
and_(MediaAsset.source_kind == "upload", MediaAsset.rel_path.is_not(None)),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -67,12 +77,17 @@ def resolve(db: Session, user_id: int) -> QuotaLimits:
|
||||
|
||||
|
||||
def footprint(db: Session, user_id: int) -> int:
|
||||
"""Total bytes of the distinct ready assets this user's active jobs reference."""
|
||||
"""Total bytes of the distinct assets this user's jobs hold on disk: the ready ones their active
|
||||
jobs reference, and every upload of theirs — including one whose conversion failed, which keeps
|
||||
its original (see the module docstring)."""
|
||||
held_assets = (
|
||||
select(DownloadJob.asset_id)
|
||||
.where(
|
||||
DownloadJob.user_id == user_id,
|
||||
DownloadJob.status.in_(_HOLDING),
|
||||
or_(
|
||||
DownloadJob.status.in_(_HOLDING),
|
||||
and_(DownloadJob.job_kind == "upload", DownloadJob.status == JobStatus.ERROR),
|
||||
),
|
||||
DownloadJob.asset_id.is_not(None),
|
||||
)
|
||||
.distinct()
|
||||
@@ -80,7 +95,7 @@ def footprint(db: Session, user_id: int) -> int:
|
||||
return int(
|
||||
db.execute(
|
||||
select(func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.id.in_(held_assets), MediaAsset.status == "ready"
|
||||
MediaAsset.id.in_(held_assets), ON_DISK
|
||||
)
|
||||
).scalar()
|
||||
or 0
|
||||
@@ -111,7 +126,7 @@ def at_concurrency_limit(db: Session, user_id: int) -> bool:
|
||||
lim = resolve(db, user_id)
|
||||
if lim.unlimited:
|
||||
return False
|
||||
running = _count(db, user_id, ("running",))
|
||||
running = _count(db, user_id, (JobStatus.RUNNING,))
|
||||
return running >= lim.max_concurrent
|
||||
|
||||
|
||||
@@ -160,6 +175,18 @@ def check_upload(db: Session, user_id: int, add_bytes: int) -> None:
|
||||
raise QuotaExceeded("max_bytes", lim.max_bytes, projected)
|
||||
|
||||
|
||||
def fits_after_conversion(db: Session, user_id: int, old_bytes: int, new_bytes: int) -> bool:
|
||||
"""Whether an upload converted from `old_bytes` (what it is charged at now) to `new_bytes` keeps
|
||||
its owner within the cap. A re-encode can come out larger than the original, and nothing else
|
||||
re-checks it once the upload was let in."""
|
||||
if new_bytes <= old_bytes: # a smaller file never breaks a cap the larger one was let in under
|
||||
return True
|
||||
lim = resolve(db, user_id)
|
||||
if lim.unlimited:
|
||||
return True
|
||||
return footprint(db, user_id) - old_bytes + new_bytes <= lim.max_bytes
|
||||
|
||||
|
||||
def usage(db: Session, user_id: int) -> dict:
|
||||
"""Compact usage snapshot for the UI (footprint + counts vs limits)."""
|
||||
lim = resolve(db, user_id)
|
||||
|
||||
@@ -95,12 +95,14 @@ def _run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id) ->
|
||||
(`cause=child_exit`)."""
|
||||
# Imported here (not at module top) so the child pays the yt-dlp import cost only when it has
|
||||
# real work, and to keep the import graph obvious: runner is a thin entry over the pipeline.
|
||||
from app.downloads.pipeline import _download, _process_edit
|
||||
from app.downloads.pipeline import _convert_upload, _download, _process_edit
|
||||
|
||||
with formats.hiding(source_ref):
|
||||
try:
|
||||
if job_kind == "edit":
|
||||
_process_edit(job_id, asset_id)
|
||||
elif job_kind == "upload":
|
||||
_convert_upload(job_id, asset_id)
|
||||
else:
|
||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
||||
|
||||
@@ -318,14 +318,22 @@ def release_asset(db: Session, job: DownloadJob) -> None:
|
||||
re-enqueuing the same (source,format) — get_or_create_asset would reset that row to `pending` and
|
||||
+1 it, and deleting it under that would FK-null the new job's asset (a lost update on ref_count).
|
||||
A ready asset is still freed at ref==0 (its file is the point); an orphaned errored row is cheap
|
||||
(no file) and gets reused+reset by the next enqueue, or reclaimed by a later GC pass."""
|
||||
(no file) and gets reused+reset by the next enqueue, or reclaimed by a later GC pass.
|
||||
|
||||
An UPLOAD is the exception: it is never shared or re-enqueued (its ref is unique), and while it
|
||||
waits for or failed its phone-safe conversion the original file already sits at `rel_path` — so
|
||||
it is freed at ref==0 whatever its state, except mid-conversion (`downloading`: the worker child
|
||||
still reads it; the GC upload pass takes it once the child has stopped)."""
|
||||
if not job.asset_id or job.status not in HOLDING:
|
||||
return
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
if asset is None:
|
||||
return
|
||||
new_count = decr_ref_count(db, asset.id)
|
||||
if new_count == 0 and asset.status == "ready":
|
||||
freeable = asset.status == "ready" or (
|
||||
asset.source_kind == "upload" and asset.status != "downloading"
|
||||
)
|
||||
if new_count == 0 and freeable:
|
||||
if asset.rel_path:
|
||||
storage.delete_asset_files(settings.download_root, asset.rel_path)
|
||||
db.delete(asset)
|
||||
|
||||
@@ -168,13 +168,25 @@ def upload_rel_path(user_id: int, asset_id: int, filename: str) -> str:
|
||||
episode sidecars). The name is system-decided and bound to the asset id (`_a{id}` keeps two
|
||||
same-named uploads apart); the original filename is display metadata only. The extension is
|
||||
preserved (shell-safe, bounded) so the served file + its Content-Disposition carry a correct
|
||||
suffix; an extensionless upload stays extensionless."""
|
||||
stem = sanitize(Path(filename).stem, 80)
|
||||
suffix; an extensionless upload stays extensionless.
|
||||
|
||||
The stem holds no dot, so the last dot of the name is always the extension's: every sibling
|
||||
derived by swapping it (`with_suffix` — the poster, the converted file) keeps `_a{id}`. A dotted
|
||||
stem with no extension ('Holiday.2024.ä') once lost it, and two such uploads shared one file."""
|
||||
stem = sanitize(Path(filename).stem, 80).replace(".", "_")
|
||||
ext = _clean_upload_ext(filename)
|
||||
suffix = f".{ext}" if ext else ""
|
||||
return f".uploads/{user_id}/{stem}_a{asset_id}{suffix}"
|
||||
|
||||
|
||||
def with_ext(rel: str, old_ext: str | None, ext: str) -> str:
|
||||
"""`rel` with its extension — `old_ext`, the stored `container`, None for a bare name — swapped
|
||||
for `ext`. Not `Path.with_suffix`: on a bare name whose stem holds a dot (an upload placed before
|
||||
the stems lost their dots) that would take the stem's tail for the extension, and `_a{id}` with it."""
|
||||
base = rel[: -len(old_ext) - 1] if old_ext and rel.endswith(f".{old_ext}") else rel
|
||||
return f"{base}.{ext}"
|
||||
|
||||
|
||||
def upload_ext(filename: str) -> str | None:
|
||||
"""The stored `container` for an uploaded file: its shell-safe lowercased extension, or None when
|
||||
the upload has no extension (a bare name). Shares `_clean_upload_ext` with `upload_rel_path` so the
|
||||
|
||||
@@ -1,15 +1,24 @@
|
||||
"""Local file ingest for the download center (E7 S5) — the non-yt-dlp path into the media library.
|
||||
"""Local file ingest for the download center — the non-yt-dlp path into the media library.
|
||||
|
||||
A user-uploaded file joins the SAME library as a downloaded one, so it inherits the whole function
|
||||
set already built (metadata chips, share links, and — for media — in-app play/edit). Unlike a
|
||||
download it bypasses yt-dlp AND the worker entirely: the file is already on the host, so ingest
|
||||
probes it, places it on disk, and creates an already-`ready` MediaAsset + `done` DownloadJob in one
|
||||
step — the same "the file exists, mark it done now" shortcut `service.enqueue` takes on a cache hit.
|
||||
download it bypasses yt-dlp: the file is already on the host, so ingest probes it and places it on
|
||||
disk. A file that already is the phone-safe shape (`playable`) — or is not media at all — becomes an
|
||||
already-`ready` MediaAsset + `done` DownloadJob in one step, the same "the file exists, mark it done
|
||||
now" shortcut `service.enqueue` takes on a cache hit.
|
||||
|
||||
Media that is NOT phone-safe (an iPhone's HEVC .mov, a VP9 .webm, an .mkv) is placed as it came and
|
||||
queued: the asset stays `pending` and the job `queued`, and the download worker converts it
|
||||
(`pipeline._convert_upload`) — so a share link never hands a friend a file their phone cannot play.
|
||||
Only the converted file is kept (the owner's choice: one file per upload). A 10-bit HDR original
|
||||
becomes 8-bit H.264 with its colour tags kept — still tagged HDR (HLG/PQ, BT.2020), not tone-mapped
|
||||
to SDR (the owner's choice too). Until then the original sits at the asset's `rel_path`, which is
|
||||
what a failed conversion's Retry starts from again, and what a delete removes
|
||||
(`service.release_asset`).
|
||||
|
||||
Kinds (the invariant lives in `mediakind`): a probe with a real video stream -> video, audio-only
|
||||
-> audio, neither -> a generic 'file' (pdf/zip/docx/…) that is download-only and never rendered
|
||||
inline (see the force-download serving in the file/watch routes). This module is the ingest seam a
|
||||
future sub-function (e.g. convert) plugs a new adapter into, mirroring how it sidesteps the worker.
|
||||
inline (see the force-download serving in the file/watch routes).
|
||||
|
||||
An upload does NOT get an `expires_at`: it's user-provided and may be irreplaceable (there's no
|
||||
source to re-download), so it doesn't age out on the retention clock the way a re-fetchable download
|
||||
@@ -24,7 +33,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import mediakind, probe, quota, service, storage
|
||||
from app.downloads import mediakind, playable, probe, quota, service, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, MediaAsset, display_text
|
||||
|
||||
|
||||
@@ -44,15 +54,22 @@ def ingest_upload(
|
||||
size: int,
|
||||
display_name: str | None = None,
|
||||
) -> DownloadJob:
|
||||
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return the `done`
|
||||
job. `src_path` is MOVED into place on success (the caller stages it to a temp file first).
|
||||
"""Ingest an already-uploaded file at `src_path` into `user_id`'s library and return its job —
|
||||
`done`, or `queued` for the worker's phone-safe conversion (see the module docstring). `src_path`
|
||||
is MOVED into place on success (the caller stages it to a temp file first).
|
||||
|
||||
Raises quota.QuotaExceeded (holdings/footprint cap) — checked BEFORE any disk write, so a
|
||||
rejected upload leaves nothing behind and the caller just cleans up its temp file."""
|
||||
quota.check_upload(db, user_id, size) # the file's size is known → charge it up front
|
||||
|
||||
vcodec, acodec, width, height, duration = probe.probe_media(src_path)
|
||||
data = probe.probe_json(src_path)
|
||||
vcodec, acodec, width, height, duration = probe.facts(data)
|
||||
kind = mediakind.derive(vcodec, acodec)
|
||||
# Judged by the extension the file is PLACED under (and the worker re-judges), not the raw one:
|
||||
# 'clip.mp4~' is stored as .mp4, and judging 'mp4~' queued a conversion with nothing to convert.
|
||||
ext = storage.upload_ext(filename) or ""
|
||||
safe = playable.plan(data, ext) if kind != mediakind.FILE else None
|
||||
convert = safe is not None and safe.needs_change
|
||||
|
||||
# An upload never dedups (each file is its own asset): a random ref makes the cache key unique,
|
||||
# so `format_sig` is a constant marker rather than a format hash.
|
||||
@@ -66,17 +83,18 @@ def ingest_upload(
|
||||
rel = storage.upload_rel_path(user_id, asset.id, filename)
|
||||
storage.place_file(src_path, settings.download_root, rel)
|
||||
# From here the file is on disk under `.uploads/`. If the transaction fails before it commits, the
|
||||
# asset/job rows roll back but the placed file would be ORPHANED — nothing references it, the
|
||||
# per-user delete path keys off asset.rel_path (which is gone), and the GC orphan sweep skips the
|
||||
# dot-prefixed `.uploads/` tree. So roll back the on-disk side too on any failure.
|
||||
# asset/job rows roll back and nothing names the placed file: the per-user delete path keys off
|
||||
# asset.rel_path (which is gone), and the GC's `.uploads` sweep reaches it only an hour later. So
|
||||
# roll back the on-disk side too — unless the commit itself raised after it landed (see below).
|
||||
asset_id = asset.id
|
||||
try:
|
||||
final = storage.abs_path(settings.download_root, rel)
|
||||
actual_size = final.stat().st_size if final.exists() else size
|
||||
|
||||
asset.status = "ready"
|
||||
asset.status = "pending" if convert else "ready"
|
||||
asset.rel_path = rel
|
||||
asset.size_bytes = actual_size
|
||||
asset.container = storage.upload_ext(filename)
|
||||
asset.container = ext or None
|
||||
asset.vcodec = vcodec
|
||||
asset.acodec = acodec
|
||||
asset.width = width
|
||||
@@ -100,8 +118,8 @@ def ingest_upload(
|
||||
source_ref=ref,
|
||||
profile_snapshot={},
|
||||
display_name=display_text(display_name),
|
||||
status="done",
|
||||
progress=100,
|
||||
status=JobStatus.QUEUED if convert else JobStatus.DONE,
|
||||
progress=0 if convert else 100,
|
||||
queue_pos=service.next_queue_pos(db),
|
||||
)
|
||||
db.add(job)
|
||||
@@ -109,6 +127,11 @@ def ingest_upload(
|
||||
db.commit()
|
||||
except Exception:
|
||||
db.rollback()
|
||||
if _row_landed(db, asset_id) is not False:
|
||||
# The commit reached the server and lost its answer, or the row cannot be read: the file
|
||||
# may be the asset's now, so it is never removed blind — the sweep takes it if no row
|
||||
# names it.
|
||||
raise
|
||||
# The rows are gone; remove the now-unreferenced file so no bytes leak. Clean the poster by its
|
||||
# DETERMINISTIC path (rel with a .jpg suffix, the same path ensure_poster writes to) rather than
|
||||
# ensure_poster's return value: it writes the frame directly to that path and returns None on an
|
||||
@@ -120,3 +143,13 @@ def ingest_upload(
|
||||
raise
|
||||
db.refresh(job)
|
||||
return job
|
||||
|
||||
|
||||
def _row_landed(db: Session, asset_id: int) -> bool | None:
|
||||
"""After the ingest transaction raised: is the asset row there anyway? A commit can reach the
|
||||
server and lose its answer (the connection drops after COMMIT). None when it cannot be read."""
|
||||
try:
|
||||
return db.get(MediaAsset, asset_id) is not None
|
||||
except Exception: # noqa: BLE001 — "cannot tell" is the answer, the caller acts on it
|
||||
# silent-ok: the caller re-raises the ingest's own error.
|
||||
return None
|
||||
|
||||
@@ -197,7 +197,7 @@ def _next_step(verify_link: str | None, has_account: bool, hours: int, lang: str
|
||||
|
||||
Three states, because a mail that guesses wrong sends someone looking for a link that isn't
|
||||
coming — or worse, tells them to sign in when their address is still unconfirmed and sign-in will
|
||||
refuse them (E9.5 S4 named that state on screen; the mail must not contradict it).
|
||||
refuse them (the sign-in screen names that state; the mail must not contradict it).
|
||||
|
||||
The link branch is the point of the C option: in the common ordering — approve first, the person
|
||||
signs up later, or signs up and waits — they would otherwise have to find and open TWO separate
|
||||
@@ -253,7 +253,7 @@ def send_access_invitation(
|
||||
the closing line offers no deadline — nudging is the one thing this mail must not do.
|
||||
|
||||
Its language is the one place the app has no signal to read: the admin typed this address in, so
|
||||
nobody has ever told us how to address its owner. That is why the send button asks (E9.5 S7).
|
||||
nobody has ever told us how to address its owner. That is why the send button asks.
|
||||
"""
|
||||
admin, contact = _reply_contact(lang)
|
||||
body = mailtext.text(
|
||||
|
||||
@@ -109,7 +109,7 @@ def _tag_visible(user: User):
|
||||
# The pseudo-category of a selected tag id no Tag row answers (a saved view or the feed state outlives
|
||||
# a deleted tag). It filters like a category whose tags no channel carries — to nothing, in the SQL and
|
||||
# in feed_summary's mirror alike — instead of the id being dropped, which showed the WHOLE feed under
|
||||
# the view's old name (E32 S2). No real category is called this (language/topic/other).
|
||||
# the view's old name. No real category is called this (language/topic/other).
|
||||
MISSING_TAG_CATEGORY = "missing"
|
||||
|
||||
|
||||
@@ -118,7 +118,7 @@ def tags_by_category(db: Session, user: User, tag_ids: list[int]) -> dict[str, l
|
||||
shared by the feed's tag WHERE (filtered_query) and the facet/count summary (feed_summary), so the
|
||||
two can't disagree on which selected tag belongs to which category. An unknown id lands in
|
||||
MISSING_TAG_CATEGORY — and so does another user's private tag: resolving it to its real category
|
||||
told the caller which foreign ids exist (E32 review)."""
|
||||
told the caller which foreign ids exist."""
|
||||
if not tag_ids:
|
||||
return {} # skip the pointless `WHERE id IN ()` round-trip on the no-tag-filter hot path
|
||||
out: dict[str, list[int]] = {}
|
||||
@@ -463,7 +463,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
||||
|
||||
|
||||
def feed_years(db: Session, user: User, **params) -> list[dict]:
|
||||
"""Per-year video counts for the current filter context (item 8 / E8·S2): powers the year control's
|
||||
"""Per-year video counts for the current filter context: powers the year control's
|
||||
quick chips + the "more…" per-year-counts dialog. Every OTHER filter (tags, channels, content type,
|
||||
search, watch state, scope) still applies, but the publish-date narrowing itself is DROPPED —
|
||||
published_after/before and max_age_days — so the year list is the full set of years available and
|
||||
@@ -559,7 +559,7 @@ def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) ->
|
||||
return table.get(sort) or table["newest"]
|
||||
|
||||
|
||||
# Relevance ranks only the newest RANK_WINDOW matches (E31): ts_rank reads every match's tsvector, and
|
||||
# Relevance ranks only the newest RANK_WINDOW matches: ts_rank reads every match's tsvector, and
|
||||
# a broad word matches most of the library ("a" ranked 193k rows in 6.8 s on prod, against 1.1 s to just
|
||||
# find them). The rows past the window keep rank 0, so they follow the ranked ones newest first; a word
|
||||
# that broad has no meaningful relevance order anyway, and a specific one fits the window whole.
|
||||
@@ -671,7 +671,7 @@ def encode_cursor(keys: list[dict], row, window: datetime | None = None) -> str:
|
||||
def _cursor_dt(v):
|
||||
"""A cursor's timestamp: aware or refused. The server writes only aware ones (timestamptz), and
|
||||
windowed_relevance_rows compares them in Python, where a naive one beside an aware one raises — a
|
||||
500 for a hand-made cursor instead of the 400 (E31 review 2)."""
|
||||
500 for a hand-made cursor instead of the 400."""
|
||||
if v is None:
|
||||
return None
|
||||
dt = datetime.fromisoformat(v)
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""Bug reports built from debug sessions (E16): the files an issue carries, and where they go —
|
||||
downloaded as a zip, or (S4) sent to a Gitea issue tracker. The session itself — what is recorded
|
||||
"""Bug reports built from debug sessions: the files an issue carries, and where they go —
|
||||
downloaded as a zip, or sent to a Gitea issue tracker. The session itself — what is recorded
|
||||
and tagged — is the core module `app/debugsession.py`."""
|
||||
|
||||
@@ -5,7 +5,7 @@ Gitea upload, so the two cannot drift apart.
|
||||
in their language; the server does not rewrite it);
|
||||
* `client.log` / `server.log` — the session's evidence (`debugsession.bundle`) minus the rows the
|
||||
user took out, one JSON object per line, browser rows and server rows apart. `.log`, not
|
||||
`.jsonl`: Gitea refuses that extension (measured, plans/e16 S0);
|
||||
`.jsonl`: Gitea refuses that extension (measured);
|
||||
* the screenshots, in the order they were taken.
|
||||
|
||||
Every row is already redacted twice (at capture, and again when stored), and nothing here adds a
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""The Gitea issue tracker a debug session's issue goes to (E16 S4) — measured against Gitea 1.27.1
|
||||
before a line of this was written (plans/e16-debug-issue.md, S0).
|
||||
"""The Gitea issue tracker a debug session's issue goes to — measured against Gitea 1.27.1
|
||||
before a line of this was written.
|
||||
|
||||
Configured by an admin (sysconfig `issues_gitea_url` / `_repo` / `_token`, the token encrypted at
|
||||
rest). The repository must be PRIVATE: an issue carries its reporter's log — page titles, filter
|
||||
@@ -79,7 +79,7 @@ def target(db: Session) -> dict | None:
|
||||
@contextmanager
|
||||
def _client(settings: tuple[str, str, str]) -> Iterator[tuple[httpx.Client, str]]:
|
||||
# Takes the settings already read, not the session: reading them HERE opened a transaction one
|
||||
# line after `deliver` had ended one, and held it across the calls below (deep-review round 4).
|
||||
# line after `deliver` had ended one, and held it across the calls below.
|
||||
url, repo, token = settings
|
||||
if not (url and repo and token):
|
||||
raise GiteaError("not_configured")
|
||||
@@ -203,7 +203,7 @@ def deliver(
|
||||
# call, so nothing holds a snapshot (and an idle connection) across minutes of tracker I/O.
|
||||
# The settings are read HERE and handed to `_client`, not read inside it: reading them there
|
||||
# was a SELECT one line after this commit, which reopened the transaction and held it — idle,
|
||||
# with its snapshot pinned — across the repository check and the create (deep-review round 4
|
||||
# with its snapshot pinned — across the repository check and the create (a review
|
||||
# measured it in pg_stat_activity). A comment saying otherwise sat right here.
|
||||
checkpoint()
|
||||
with _client(settings) as (client, repo):
|
||||
@@ -223,7 +223,7 @@ def deliver(
|
||||
)
|
||||
# Built ON the state read at the start so nothing already in it is dropped. The claim
|
||||
# is NOT among those things, whatever this comment used to say: `set_delivery` strips
|
||||
# any `sending` passed to it and stamps the row's own from `token` (round 4 — removing
|
||||
# any `sending` passed to it and stamps the row's own from `token` (removing
|
||||
# the spread left the whole suite green, because the claim never travelled here).
|
||||
state = {
|
||||
**state,
|
||||
|
||||
+11
-12
@@ -10,7 +10,7 @@ the logs were both unjoinable and leaky. `configure_logging()` replaces all four
|
||||
Event shape — the OpenTelemetry log data model, without the SDK; one JSON object per line:
|
||||
ts · level · sev · source · logger · event · msg · attrs
|
||||
plus whatever correlation the context carries (`request_id`, `user_id`, `debug_session_id`,
|
||||
`job_id`) and the app version. That shape is what E22.6's admin console filters on and what a Gitea
|
||||
`job_id`) and the app version. That shape is what the admin log console filters on and what a Gitea
|
||||
issue attachment will carry, so it is deliberately flat and stable.
|
||||
|
||||
Call sites stay ordinary stdlib logging:
|
||||
@@ -106,7 +106,7 @@ class _RequestScope:
|
||||
self.user_id: int | None = None
|
||||
# The user's consented diagnostics level as a logging number; None at the base level.
|
||||
self.level: int | None = None
|
||||
# The user's own open debug session this request named (E16), else None.
|
||||
# The user's own open debug session this request named, else None.
|
||||
self.debug_session_id: str | None = None
|
||||
|
||||
|
||||
@@ -122,11 +122,11 @@ def set_user(
|
||||
debug_session: str | None = None,
|
||||
) -> None:
|
||||
"""Attribute the rest of this request's log lines to `user_id`, and log them at the user's
|
||||
consented diagnostics `level` until `until` (E22 S5; "INFO" = no elevation). Called from the
|
||||
consented diagnostics `level` until `until` ("INFO" = no elevation). Called from the
|
||||
auth dependency — the one place the session is actually resolved into an account — rather than
|
||||
from the request middleware, which runs OUTSIDE the session middleware.
|
||||
|
||||
`debug_session` tags every line with the user's open debug session (E16). The caller has
|
||||
`debug_session` tags every line with the user's open debug session. The caller has
|
||||
already checked it is theirs and open; a session's level arrives here as `level`/`until`.
|
||||
|
||||
The root logger's floor is tracked per elevation: a request without the session's header (an
|
||||
@@ -334,7 +334,7 @@ _RECORD_STANDARD = frozenset(
|
||||
class ContextFilter(logging.Filter):
|
||||
"""Copies the correlation contextvars onto the record. A Filter (not the formatter) so the
|
||||
values are captured at emit time, before a QueueHandler can move the record to another
|
||||
thread — the DB sink in S3 depends on that."""
|
||||
thread — the DB sink depends on that."""
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
for name, var in _CONTEXT_VARS.items():
|
||||
@@ -355,7 +355,7 @@ class ContextFilter(logging.Filter):
|
||||
|
||||
class LevelGate(logging.Filter):
|
||||
"""The configured level, enforced at the HANDLER, with one exception: the lines of a request
|
||||
whose user raised their diagnostics level (E22 S5).
|
||||
whose user raised their diagnostics level.
|
||||
|
||||
The root logger's level cannot do this alone: it is one number for the whole process, and the
|
||||
point is to log ONE user's requests in detail while everyone else's stay at the base level. So
|
||||
@@ -457,7 +457,7 @@ class TextFormatter(logging.Formatter):
|
||||
# --- configuration -------------------------------------------------------------------------
|
||||
# Third-party loggers that are noisy at our own level. Pinned explicitly so the diagnostics knob can
|
||||
# lower the ROOT level to DEBUG/TRACE without drowning the output in SQL statements and HTTP chatter
|
||||
# — raising THESE is a deliberate, separate act (E22.6's console).
|
||||
# — raising THESE is a deliberate, separate act (the admin log console).
|
||||
_THIRD_PARTY_LEVELS = {
|
||||
"sqlalchemy": "WARNING",
|
||||
"sqlalchemy.engine": "WARNING",
|
||||
@@ -481,7 +481,7 @@ def get_logger(name: str) -> logging.Logger:
|
||||
`siftlode.plex.sync`.
|
||||
|
||||
The names were literals before, and 25 of them covered 34 modules — six different Plex modules
|
||||
all called themselves `siftlode.plex`, so neither a level pinned per logger nor E22.6's console
|
||||
all called themselves `siftlode.plex`, so neither a level pinned per logger nor the console's
|
||||
filter could tell a sync line from a stream line. Derived from `__name__` there is exactly one
|
||||
name per module, it cannot drift from the file it sits in, and `test_logger_names.py` fails if a
|
||||
module hand-rolls one again.
|
||||
@@ -562,12 +562,12 @@ def current_level() -> str:
|
||||
return logging.getLevelName(_base_level or logging.getLogger().getEffectiveLevel())
|
||||
|
||||
|
||||
# --- per-user elevation (E22 S5) -----------------------------------------------------------
|
||||
# --- per-user elevation -----------------------------------------------------------
|
||||
# The base level (`LOG_LEVEL`, then the admin knob). NOTSET until configure_logging runs — a unit
|
||||
# test, a script — so the gate lets everything through there, as it did before the gate existed.
|
||||
_base_level = logging.NOTSET
|
||||
# → (level number, expiry) for every elevation this process has seen and not yet expired: a user's
|
||||
# own, by user id, and a debug session's (E16), by (user id, session id) — its requests alone carry
|
||||
# own, by user id, and a debug session's, by (user id, session id) — its requests alone carry
|
||||
# its level, so it is tracked apart from the user's own requests.
|
||||
_ElevationKey = int | tuple[int, str]
|
||||
_elevations: dict[_ElevationKey, tuple[int, datetime]] = {}
|
||||
@@ -667,7 +667,7 @@ def current_source() -> str:
|
||||
|
||||
|
||||
# --- request logging -----------------------------------------------------------------------
|
||||
# A client may supply its own request id so a browser log line and a server line join up (E22 S4).
|
||||
# A client may supply its own request id so a browser log line and a server line join up.
|
||||
# Accepted only in this shape: it ends up in a log field, and an unbounded header would be a way to
|
||||
# write arbitrary text into the log.
|
||||
_REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._-]{8,64}$")
|
||||
@@ -688,7 +688,6 @@ _POLLED_ROUTES = frozenset(
|
||||
{
|
||||
"/api/me",
|
||||
"/api/me/notifications/unread_count",
|
||||
"/api/messages/unread_count",
|
||||
"/api/downloads",
|
||||
"/api/downloads/index",
|
||||
"/api/downloads/usage",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The `log_events` sink (E22 S3): the stored copy of the log the diagnostics console reads.
|
||||
"""The `log_events` sink: the stored copy of the log the diagnostics console reads.
|
||||
|
||||
stdout stays the complete log (docker keeps 3×50 MB of it); this table keeps the part worth querying
|
||||
for `log_retention_days`:
|
||||
@@ -6,7 +6,7 @@ for `log_retention_days`:
|
||||
* every WARNING and above,
|
||||
* the `http.request` line of every request at INFO or above (not the DEBUG-level health checks and
|
||||
static files),
|
||||
* every line of a request whose user raised their diagnostics level (E22 S5 — `logs.LevelGate`
|
||||
* every line of a request whose user raised their diagnostics level (`logs.LevelGate`
|
||||
marks them), which is the detail they consented to have kept,
|
||||
* nothing a caller marked `extra={logs.PERSIST: False}` — how the request log caps anonymous
|
||||
traffic per client.
|
||||
@@ -347,8 +347,8 @@ def gc(db: Session, retention_days: int) -> int:
|
||||
|
||||
|
||||
def serialize(row: LogEvent) -> dict:
|
||||
"""A stored row as the API sends it — one shape for every reader: the admin console (E22.6) and
|
||||
a debug session's evidence list (E16)."""
|
||||
"""A stored row as the API sends it — one shape for every reader: the admin console and
|
||||
a debug session's evidence list."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"ts": iso(row.ts),
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Bilingual text for every outbound mail, and the one resolver that decides which language a
|
||||
recipient gets (E9.5 S7).
|
||||
recipient gets.
|
||||
|
||||
Why a catalog instead of literals in `email.py`: HU/EN parity is a house rule for every user-facing
|
||||
string, and a mail is the one user-facing surface the reader CANNOT re-render by switching the UI
|
||||
@@ -53,7 +53,7 @@ def lang_for_user(user: User | None) -> str:
|
||||
|
||||
def lang_for_email(db: Session, email: str) -> str:
|
||||
"""The language for an address that may not have an account yet: their own account preference,
|
||||
else the language they filled our access-request form in, else English.
|
||||
else the language their access request came in (the locale Google reported), else English.
|
||||
|
||||
Matched CANONICALLY like every other identity lookup in this epic, and it walks past rows that
|
||||
carry no usable code instead of stopping at the first one — prod really does hold two accounts
|
||||
@@ -82,8 +82,8 @@ def lang_for_email(db: Session, email: str) -> str:
|
||||
# a name, not a translated string, and giving them keys would invite someone to "translate" them.
|
||||
MAIL_TEXT: dict[str, dict[str, str]] = {
|
||||
# ── How long a link lasts ───────────────────────────────────────────────────────────────
|
||||
# Rendered, not interpolated as a raw hour count: the approval mail's token lives seven days
|
||||
# (E9.5 S6), and "expires in 168 hours" is a unit no reader converts.
|
||||
# Rendered, not interpolated as a raw hour count: the approval mail's token lives seven days,
|
||||
# and "expires in 168 hours" is a unit no reader converts.
|
||||
"duration.hours": {"en": "{n} hours", "hu": "{n} óra"},
|
||||
"duration.days": {"en": "{n} days", "hu": "{n} nap"},
|
||||
# ── Shared closing fragments ────────────────────────────────────────────────────────────────
|
||||
|
||||
+14
-10
@@ -44,7 +44,6 @@ from app.routes import ( # noqa: E402 — logging is configured first
|
||||
feed,
|
||||
health,
|
||||
me,
|
||||
messages,
|
||||
notifications,
|
||||
playlists,
|
||||
plex as plex_routes,
|
||||
@@ -132,7 +131,7 @@ async def _reap_hls_sessions() -> None:
|
||||
A FINISHED session is reaped on the short rule here, not kept for the full idle window: an open
|
||||
player checkpoints every 10 s and that route touches the session, so a paused viewer is no longer
|
||||
invisible to this pass (see `reap_idle`). Keeping every finished session for ten minutes instead
|
||||
held one whole segmented copy of a film per item on the download volume (round 5)."""
|
||||
held one whole segmented copy of a film per item on the download volume."""
|
||||
while True:
|
||||
await asyncio.sleep(_HLS_REAP_EVERY_S)
|
||||
try:
|
||||
@@ -188,8 +187,8 @@ async def _nul_in_text(request: Request, exc: DataError) -> JSONResponse:
|
||||
NUL in text, so the driver refuses to even send the query — and a `%00` in a query parameter or
|
||||
an escaped NUL in a JSON body (a playlist's `contains=`, a reorder's key list, a collection
|
||||
search) was a 500 with a traceback wherever it reached a comparison. Guarding each site did not
|
||||
hold: round 3 guarded the by-rating_key lookup and missed the rest. Any other DataError is left
|
||||
alone — re-raised, it stays the 500 it always was."""
|
||||
hold: an earlier fix guarded the by-rating_key lookup and missed the rest. Any other DataError
|
||||
is left alone — re-raised, it stays the 500 it always was."""
|
||||
if "NUL (0x00)" not in str(exc.orig):
|
||||
raise exc
|
||||
return JSONResponse(
|
||||
@@ -311,7 +310,6 @@ app.include_router(me.router)
|
||||
app.include_router(diagnostics_routes.router)
|
||||
app.include_router(debug_sessions_routes.router)
|
||||
app.include_router(notifications.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(channels.router)
|
||||
app.include_router(playlists.router)
|
||||
app.include_router(saved_views.router)
|
||||
@@ -341,6 +339,8 @@ STATIC_DIR = Path(__file__).parent / "static_spa"
|
||||
# stay cached but forces revalidation (cheap 304 via the FileResponse ETag) on every load. The
|
||||
# hashed bundles under /assets can cache forever (a new build changes their filename).
|
||||
INDEX_HTML = STATIC_DIR / "index.html"
|
||||
# A public share link's own page: the player alone, without the app's bundle (frontend/watch.html).
|
||||
WATCH_HTML = STATIC_DIR / "watch.html"
|
||||
INDEX_HEADERS = {"Cache-Control": "no-cache"}
|
||||
app.mount(
|
||||
"/assets",
|
||||
@@ -356,16 +356,16 @@ async def index() -> FileResponse:
|
||||
|
||||
@app.get("/watch/{token}")
|
||||
async def watch_page(token: str):
|
||||
"""Serve the SPA for a public share link, but with per-video Open Graph tags injected so the
|
||||
"""Serve the public player page for a share link, with per-video Open Graph tags injected so the
|
||||
link unfurls richly (title/channel/thumbnail) in Messenger/social crawlers. A real browser
|
||||
ignores the extra tags and hydrates WatchPage as usual; invalid/expired/password links fall
|
||||
back to the generic card."""
|
||||
ignores the extra tags and boots WatchPage; invalid/expired/password links fall back to the
|
||||
generic card."""
|
||||
from app.downloads import og
|
||||
|
||||
with SessionLocal() as db:
|
||||
rendered = og.render_watch_html(token, INDEX_HTML, db)
|
||||
rendered = og.render_watch_html(token, WATCH_HTML, db)
|
||||
if rendered is None:
|
||||
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|
||||
return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
|
||||
return HTMLResponse(rendered, headers=INDEX_HEADERS)
|
||||
|
||||
|
||||
@@ -383,4 +383,8 @@ async def spa_fallback(full_path: str) -> FileResponse:
|
||||
# Real SPA-root assets (welcome images, favicon, robots.txt) rarely change and have
|
||||
# stable names, so a moderate cache is safe and satisfies the cache-lifetime audit.
|
||||
return FileResponse(candidate, headers={"Cache-Control": "public, max-age=604800"})
|
||||
# A /watch/ path the route above does not match (a trailing segment, no token) is still the
|
||||
# player's: WatchPage reads the token off the path and says when a link is gone.
|
||||
if full_path.startswith("watch/"):
|
||||
return FileResponse(WATCH_HTML, headers=INDEX_HEADERS)
|
||||
return FileResponse(INDEX_HTML, headers=INDEX_HEADERS)
|
||||
|
||||
+58
-118
@@ -59,7 +59,7 @@ POSITION_MAX_S = 2**31 - 1
|
||||
|
||||
class TimestampMixin:
|
||||
"""Adds a server-defaulted `created_at`. Models that index created_at or order it
|
||||
differently (QuotaEvent, Notification, Message) keep their own column on purpose."""
|
||||
differently (QuotaEvent, Notification) keep their own column on purpose."""
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
@@ -101,15 +101,15 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
# Google sign-in that failed to find it), and a migration must not merge accounts on its own.
|
||||
email_canonical: Mapped[str] = mapped_column(String(320), index=True, server_default="")
|
||||
# The address Google asserted for this account, recorded at sign-in / link time. NOT the same
|
||||
# thing as `email`: since E9.5 S3 a Google sign-in no longer overwrites the address the user
|
||||
# thing as `email`: a Google sign-in no longer overwrites the address the user
|
||||
# registered with, so a deliberate `me+siftlode@gmail.com` survives — and this is where the other
|
||||
# half of the truth lives, for the UI to show when the two differ. NULL until the next Google
|
||||
# sign-in for accounts linked before this column existed.
|
||||
google_email: Mapped[str | None] = mapped_column(String(320))
|
||||
# argon2id hash for email+password login; NULL for Google-only / demo accounts.
|
||||
# argon2id hash for email+password login; NULL for Google-only accounts.
|
||||
password_hash: Mapped[str | None] = mapped_column(String(255))
|
||||
# When SEC-10 dropped a preset password on Google adoption (see the single caller of
|
||||
# auth.revoke_preset_password_if_unverified; admin approval no longer revokes — E9.5 S1). Without
|
||||
# auth.revoke_untrusted_preset_password; admin approval no longer revokes). Without
|
||||
# this trace a revoked account was indistinguishable from a Google-only one, so it failed the
|
||||
# password-reset eligibility check and could never be recovered — it could neither sign in nor
|
||||
# reset. Set = "had a password, lost it to the guard" -> reset-eligible. Cleared whenever a
|
||||
@@ -123,7 +123,7 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
# Admin-imposed access block, distinct from is_active (approval lifecycle). A suspended
|
||||
# account can't sign in by any method and its live sessions are rejected; the user is told
|
||||
# why and pointed at the operator's contact. The demo account is never suspendable.
|
||||
# why and pointed at the operator's contact.
|
||||
is_suspended: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Server-side session revocation (SA4): a monotonic counter bumped on password reset, password
|
||||
# change, or "log out everywhere". A signed session cookie records the epoch it was minted under;
|
||||
@@ -132,9 +132,9 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
session_epoch: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
# When this account last signed in (Google or password), stamped by the one sign-in path,
|
||||
# routes/auth._establish_session. NULL = not since the column existed (0085; no backfill is
|
||||
# possible). The shared demo account is never stamped — its "last login" would mean nobody.
|
||||
# possible).
|
||||
last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# The diagnostics level this user consented to (E22 S5) and when it lapses back to the base
|
||||
# The diagnostics level this user consented to and when it lapses back to the base
|
||||
# level. Both NULL = base. Written only by `PUT /api/me/diagnostics`, which computes the expiry
|
||||
# itself; read through `app.diagnostics.effective_level`, never directly (an expired level is
|
||||
# not cleared eagerly — it simply stops counting).
|
||||
@@ -146,12 +146,6 @@ class User(Base, TimestampMixin, CanonicalEmailMixin):
|
||||
# The user's own YouTube channel id, resolved lazily (channels.list?mine=true) and
|
||||
# cached. Used to exclude their own channel from playlist discovery. NULL = unresolved.
|
||||
yt_channel_id: Mapped[str | None] = mapped_column(String(32))
|
||||
# The single shared "demo" account: signed into without Google OAuth (via a whitelisted
|
||||
# email on the login page), has no OAuth token / YouTube scope, and its per-user state is
|
||||
# shared by everyone who enters this way. Exactly one such row is expected.
|
||||
is_demo: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
# Free-form UI preferences (theme, color scheme, font scale, default filters…).
|
||||
preferences: Mapped[dict | None] = mapped_column(JSON)
|
||||
|
||||
@@ -218,25 +212,12 @@ class Invite(Base, CanonicalEmailMixin):
|
||||
# The language this person asked us in ("en"/"hu"), captured when the request was made: the page
|
||||
# their form was on, or the locale Google reported for a refused sign-in. NULL for a row the
|
||||
# admin whitelisted by hand — nobody has told us anything about that mailbox, which is exactly
|
||||
# why the invitation send asks (E9.5 S7). Captured rather than derived for the same reason
|
||||
# why the invitation send asks. Captured rather than derived for the same reason
|
||||
# `preferences.language` and `google_email` are: the signal exists only in that one moment and
|
||||
# cannot be backfilled afterwards.
|
||||
lang: Mapped[str | None] = mapped_column(String(8))
|
||||
|
||||
|
||||
class DemoWhitelist(Base, TimestampMixin):
|
||||
"""Emails that may enter the shared demo account from the login page (no Google OAuth).
|
||||
A hidden, admin-curated list of "keys to the same door" — every entry logs into the one
|
||||
shared demo user. Distinct from Invite (which gates real Google sign-in)."""
|
||||
|
||||
__tablename__ = "demo_whitelist"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True, index=True)
|
||||
note: Mapped[str | None] = mapped_column(Text)
|
||||
added_by: Mapped[str | None] = mapped_column(String(320)) # admin email
|
||||
|
||||
|
||||
class Channel(Base, TimestampMixin):
|
||||
"""A YouTube channel. Shared across all users (one channel's videos are the same
|
||||
for everyone), so its expensive metadata is fetched and stored only once."""
|
||||
@@ -481,6 +462,27 @@ class VideoState(Base, UpdatedAtMixin):
|
||||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class LibraryState(Base):
|
||||
"""Where a user stopped in a library item (their own download, or one shared with them) — the
|
||||
library player's resume point. Keyed by the JOB, not the file: a job is the user's library entry,
|
||||
a re-download keeps it, and deleting the job deletes the row; the shared file under it may outlive
|
||||
both. A revoked share leaves its recipient's row behind, unread — only a job the user can still
|
||||
reach is ever read or written — and it is back if the item is shared again. Rows exist only for a
|
||||
position worth keeping (see `routes._common.kept_position`); no row means "from the start"."""
|
||||
|
||||
__tablename__ = "library_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "job_id", name="uq_library_user_job"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
||||
position_seconds: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
progress_updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
|
||||
|
||||
class SearchFind(Base, TimestampMixin):
|
||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||
|
||||
@@ -586,7 +588,7 @@ class AuditLog(Base):
|
||||
|
||||
|
||||
class LogEvent(Base):
|
||||
"""One stored log event (E22 S3) — the `log_events` table the diagnostics console reads. Rows are
|
||||
"""One stored log event — the `log_events` table the diagnostics console reads. Rows are
|
||||
written by app/logsink.py's background writer, never inside a request's transaction, already
|
||||
redacted (the same `logs.record_fields` the stdout line is built from).
|
||||
|
||||
@@ -599,7 +601,7 @@ class LogEvent(Base):
|
||||
__tablename__ = "log_events"
|
||||
__table_args__ = (
|
||||
Index("ix_log_events_user_ts", "user_id", "ts"),
|
||||
# E16: an issue's bundle is `WHERE debug_session_id = …`; nearly every row has none.
|
||||
# An issue's bundle is `WHERE debug_session_id = …`; nearly every row has none.
|
||||
Index(
|
||||
"ix_log_events_debug_session",
|
||||
"debug_session_id",
|
||||
@@ -625,7 +627,7 @@ class LogEvent(Base):
|
||||
|
||||
|
||||
class DebugSession(Base):
|
||||
"""One recording: Start → reproduce → Stop (E16, app/debugsession.py). While it is open, every
|
||||
"""One recording: Start → reproduce → Stop (app/debugsession.py). While it is open, every
|
||||
request of its user that names it (`X-Debug-Session`) is logged at `level` and tagged with `id`,
|
||||
and the issue it ends in is built from the rows carrying that tag. `until` is the hard cap the
|
||||
server set at Start; an open session past it is over whether or not the sweep has closed it yet.
|
||||
@@ -650,12 +652,12 @@ class DebugSession(Base):
|
||||
# app/debugsession.py `EndReason`: stopped by the user, or past `until`.
|
||||
ended_reason: Mapped[str | None] = mapped_column(String(16))
|
||||
issue_url: Mapped[str | None] = mapped_column(String(512))
|
||||
# The issue being written from it (E16 S3; app/debugsession.py `save_draft` holds its shape).
|
||||
# The issue being written from it (app/debugsession.py `save_draft` holds its shape).
|
||||
draft: Mapped[dict | None] = mapped_column(JSONB)
|
||||
|
||||
|
||||
class DebugScreenshot(Base):
|
||||
"""A screenshot attached to a debug session (E16) — pasted, dropped, or captured from the pill.
|
||||
"""A screenshot attached to a debug session — pasted, dropped, or captured from the pill.
|
||||
Stored in the row: bounded per session (app/debugsession.py), and gone with it by cascade."""
|
||||
|
||||
__tablename__ = "debug_screenshots"
|
||||
@@ -814,44 +816,6 @@ class Notification(Base):
|
||||
)
|
||||
|
||||
|
||||
class Message(Base):
|
||||
"""A message in the notification module (phase 2).
|
||||
|
||||
Two kinds share this table:
|
||||
- `kind="user"`: a private, END-TO-END ENCRYPTED direct message between two users. The
|
||||
server only ever stores `ciphertext` + `iv` (base64); it never holds the plaintext or
|
||||
the keys, so not even an admin can read it. A conversation is just the set of messages
|
||||
between two users — no separate thread entity; queries group by the {sender, recipient}
|
||||
pair.
|
||||
- `kind="system"`: a server-authored plaintext message (e.g. the welcome, or future admin
|
||||
announcements). `sender_id` is NULL and `body` holds the text. These are non-private
|
||||
boilerplate, readable without any E2EE key setup, and surface as a "Siftlode" conversation.
|
||||
|
||||
`read_at` is stamped when the recipient opens the thread (NULL = unread, feeds the nav
|
||||
badge). The recipient FK cascades (and sender, for user messages) so a GDPR account erasure
|
||||
removes that user's messages outright."""
|
||||
|
||||
__tablename__ = "messages"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
||||
# NULL for system messages (no human sender).
|
||||
sender_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
recipient_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# System messages carry plaintext `body`; user messages carry `ciphertext`+`iv` (base64).
|
||||
body: Mapped[str | None] = mapped_column(Text)
|
||||
ciphertext: Mapped[str | None] = mapped_column(Text)
|
||||
iv: Mapped[str | None] = mapped_column(String(32))
|
||||
read_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=func.now(), index=True
|
||||
)
|
||||
|
||||
|
||||
class MediaAsset(Base, TimestampMixin):
|
||||
"""A physical downloaded media file — the SHARED cache layer of the download center.
|
||||
|
||||
@@ -946,8 +910,8 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A user's request to download one source — the per-user layer over `MediaAsset`.
|
||||
|
||||
The worker claims `queued` jobs with ``FOR UPDATE SKIP LOCKED`` and writes live
|
||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery — the
|
||||
worker is a separate process and can't reach the in-process WS registry). `profile_snapshot`
|
||||
`progress`/`speed_bps`/`eta_s`/`phase` here (the frontend polls via useLiveQuery: there is
|
||||
no server push channel, and the worker is a separate process anyway). `profile_snapshot`
|
||||
freezes the spec at enqueue so later edits/deletion of the profile don't change this job.
|
||||
`display_name` is the user's own name for the file (display + the Content-Disposition on
|
||||
local download); the on-disk asset is never renamed. `asset_id` links to the shared file
|
||||
@@ -969,7 +933,8 @@ class DownloadJob(Base, TimestampMixin, UpdatedAtMixin):
|
||||
)
|
||||
source_kind: Mapped[str] = mapped_column(String(16)) # "youtube" | "url"
|
||||
source_ref: Mapped[str] = mapped_column(String(512))
|
||||
# "download" (default) or "edit" (a per-user trim/crop derivative of another job).
|
||||
# "download" (default), "edit" (a per-user trim/crop derivative of another job) or "upload" (a
|
||||
# user's own file, converted by the worker when it is not phone-safe — see `downloads.upload`).
|
||||
job_kind: Mapped[str] = mapped_column(String(16), default="download", server_default="download")
|
||||
# For edit jobs: the parent download this clip is cut from (SET NULL if the parent is deleted).
|
||||
source_job_id: Mapped[int | None] = mapped_column(
|
||||
@@ -1094,29 +1059,6 @@ class DownloadCredential(Base, TimestampMixin, UpdatedAtMixin):
|
||||
cookies_enc: Mapped[str] = mapped_column(Text) # Fernet ciphertext — never returned/logged
|
||||
|
||||
|
||||
class MessageKey(Base, TimestampMixin):
|
||||
"""A user's end-to-end-encryption key material for direct messaging (phase 2).
|
||||
|
||||
One row per user. `public_key` (base64 SPKI, ECDH P-256) is shared with others so they can
|
||||
derive the pairwise conversation key. `wrapped_private_key` is the user's private key
|
||||
encrypted IN THE BROWSER with a key derived (PBKDF2) from a message passphrase the server
|
||||
never sees — so the server stores only an opaque blob it cannot unwrap. `salt`/`wrap_iv`
|
||||
parameterise that derivation; `key_check` is a small ciphertext the client decrypts to
|
||||
verify a passphrase on unlock. The server is purely a key directory + blob store; it can
|
||||
never read any message."""
|
||||
|
||||
__tablename__ = "message_keys"
|
||||
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
public_key: Mapped[str] = mapped_column(Text)
|
||||
wrapped_private_key: Mapped[str] = mapped_column(Text)
|
||||
salt: Mapped[str] = mapped_column(String(64))
|
||||
wrap_iv: Mapped[str] = mapped_column(String(32))
|
||||
key_check: Mapped[str] = mapped_column(Text)
|
||||
|
||||
|
||||
# --- Plex integration (optional) -------------------------------------------------------------
|
||||
# A dedicated, self-contained catalog mirrored from a locally-reachable Plex server. Playback is
|
||||
# from the LOCAL physical file (Plex is metadata only); per-user watch state lives in plex_states
|
||||
@@ -1137,7 +1079,7 @@ class PlexLibrary(Base, TimestampMixin, UpdatedAtMixin):
|
||||
kind: Mapped[str] = mapped_column(String(16)) # "movie" | "show"
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# HU-overlay support (E5 S5). `metadata_lang` tags the section's metadata language. An OVERLAY
|
||||
# HU-overlay support. `metadata_lang` tags the section's metadata language. An OVERLAY
|
||||
# library (overlay_of set) is the Hungarian-metadata twin of another section: it is never browsable
|
||||
# on its own — its titles' HU fields are written onto the matched EN rows (paired by `guid`).
|
||||
metadata_lang: Mapped[str] = mapped_column(String(8), default="en", server_default="en")
|
||||
@@ -1166,7 +1108,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# Canonical external id (e.g. "tmdb:12345") — the key that pairs this EN row with its HU-overlay
|
||||
# twin during sync (E5 S5). Indexed for the guid->row overlay match.
|
||||
# twin during sync. Indexed for the guid->row overlay match.
|
||||
guid: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
@@ -1196,7 +1138,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
) # cast+directors, folded into search_vector
|
||||
# Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast so
|
||||
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
||||
imdb_rating: Mapped[float | None] = mapped_column(
|
||||
Float
|
||||
@@ -1206,7 +1148,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
full_cast: Mapped[list | None] = mapped_column(
|
||||
JSONB(none_as_null=True)
|
||||
) # ordered [{name, role, thumb}]
|
||||
# --- TMDB enrichment (E5 S2): see the identical block on plex_items for the full rationale. Additive;
|
||||
# --- TMDB enrichment: see the identical block on plex_items for the full rationale. Additive;
|
||||
# served from the row; NULL until enriched. (`budget`/`revenue` stay NULL for shows — TV has none.) ---
|
||||
tmdb_id: Mapped[int | None] = mapped_column(Integer)
|
||||
tmdb_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
@@ -1224,10 +1166,10 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
production_countries: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # [str]
|
||||
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
backdrops: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
||||
# Hungarian-metadata twin fields, written by the overlay sync. NULL = no HU twin matched,
|
||||
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline (E30)
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
@@ -1289,7 +1231,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
)
|
||||
kind: Mapped[str] = mapped_column(String(16), index=True) # "movie" | "episode"
|
||||
# Canonical external id (e.g. "tmdb:12345") — pairs a MOVIE row with its HU-overlay twin during
|
||||
# sync (E5 S5); indexed for the guid->row match. Episodes carry it too but it's unused for them.
|
||||
# sync; indexed for the guid->row match. Episodes carry it too but it's unused for them.
|
||||
guid: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
show_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_shows.id", ondelete="CASCADE"), index=True
|
||||
@@ -1316,7 +1258,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# Browser-playability class from ffprobe: "direct" | "remux" | "transcode" (None = unprobed).
|
||||
playable: Mapped[str | None] = mapped_column(String(12), index=True)
|
||||
probed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Transcode HW-capability facet (E7 playback-class filter). NULL for non-transcode items; for a
|
||||
# Transcode HW-capability facet (the playback-class filter). NULL for non-transcode items; for a
|
||||
# `playable="transcode"` item: True = the prod iGPU can hardware-transcode it (codec is HW-decodable
|
||||
# AND the source is 8-bit SDR), False = software libx264 only (AV1/XviD, or a 10-bit/HDR source that
|
||||
# breaks the VAAPI nv12 pipeline — see plex/stream.py). Derived by `plex.sync.transcode_hw_capable`:
|
||||
@@ -1332,8 +1274,6 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
# bit-depth/HDR. NULL = never (the backfill's work-list). Only HW-decodable transcode items are
|
||||
# probed (the rest are SW-only by codec and need no bit-depth); see `_probe_media_batch`.
|
||||
media_probed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Intro/credit markers: [{"type": "intro"|"credits", "start_s": int, "end_s": int}, ...]
|
||||
markers: Mapped[list | None] = mapped_column(JSON)
|
||||
added_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True)
|
||||
# Filterable / orderable metadata mirrored from the Plex section listing (no extra API calls;
|
||||
# `rating` = Plex audienceRating, which is ~the IMDb score). The arrays back genre/person/studio
|
||||
@@ -1352,7 +1292,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
people_text: Mapped[str | None] = mapped_column(Text)
|
||||
# Set once the movie's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast so
|
||||
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
||||
imdb_rating: Mapped[float | None] = mapped_column(
|
||||
Float
|
||||
@@ -1362,7 +1302,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
full_cast: Mapped[list | None] = mapped_column(
|
||||
JSONB(none_as_null=True)
|
||||
) # ordered [{name, role, thumb}]
|
||||
# --- TMDB enrichment (E5 S2): matched by the canonical `guid` ("tmdb:12345"), filled by the TMDB
|
||||
# --- TMDB enrichment: matched by the canonical `guid` ("tmdb:12345"), filled by the TMDB
|
||||
# backfill job, SERVED FROM THIS ROW (never a live TMDB call). Additive — imdb_rating/rating
|
||||
# above stay Plex-sourced; `tmdb_rating` is the distinct TMDb community score. NULL until enriched;
|
||||
# `tmdb_enriched_at` is the "tried" watermark (stamped on a definitive no-match too, so an
|
||||
@@ -1385,11 +1325,11 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
recommendations: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
backdrops: Mapped[list | None] = mapped_column(
|
||||
JSONB(none_as_null=True)
|
||||
) # TMDB backdrop file_paths [str] (crossfade pool, S8)
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
||||
) # TMDB backdrop file_paths [str] (crossfade pool)
|
||||
# Hungarian-metadata twin fields, written by the overlay sync. NULL = no HU twin matched
|
||||
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline (E30)
|
||||
tagline_hu: Mapped[str | None] = mapped_column(Text) # the HU overlay's tagline
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
@@ -1428,15 +1368,15 @@ class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), unique=True, index=True)
|
||||
photo_thumb: Mapped[str | None] = mapped_column(Text) # proxied /person-image URL, or None
|
||||
# TMDB person id, linked by name from a title's TMDB credits (E5 S2 lays the foundation; the /person
|
||||
# biography fetch + person-page render land in S6). NULL until a TMDB-enriched title names this person.
|
||||
# TMDB person id, linked by name from a title's TMDB credits (the /person
|
||||
# biography fetch builds on it). NULL until a TMDB-enriched title names this person.
|
||||
tmdb_person_id: Mapped[int | None] = mapped_column(Integer)
|
||||
# --- TMDB person enrichment (E5 S6): fetched from /person/{tmdb_person_id} by the people_enrich job
|
||||
# (E30 S6), or by the first person-page open the job hasn't reached, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||
# --- TMDB person enrichment: fetched from /person/{tmdb_person_id} by the people_enrich job,
|
||||
# or by the first person-page open the job hasn't reached, then SERVED FROM THE ROW. `tmdb_enriched_at` is the watermark (set even on a
|
||||
# no-op fetch so it isn't retried every open). `known_for` = top combined-credits [{tmdb_id,
|
||||
# media_type, title}], resolved to library rows by guid at serve time (like recommendations). ---
|
||||
biography: Mapped[str | None] = mapped_column(Text)
|
||||
# TMDB's Hungarian biography (E30 S4), from the same request's `translations`; a Hungarian reader is
|
||||
# TMDB's Hungarian biography, from the same request's `translations`; a Hungarian reader is
|
||||
# served it, and the instance-language `biography` when TMDB has none. Birthplace, aliases and
|
||||
# keywords have no Hungarian on TMDB, so they stay in the instance language.
|
||||
biography_hu: Mapped[str | None] = mapped_column(Text)
|
||||
@@ -1477,7 +1417,7 @@ class PlexCollection(Base, TimestampMixin, UpdatedAtMixin):
|
||||
class PlexState(Base, UpdatedAtMixin):
|
||||
"""Per-user watch state for a Plex item — mirrors VideoState. Lives in Siftlode's own DB so
|
||||
it works for users WITHOUT a Plex account (the whole point of the access-layer design).
|
||||
`synced_to_plex` marks whether this state was pushed back to a linked Plex account (P5)."""
|
||||
`synced_to_plex` marks whether this state was pushed back to a linked Plex account."""
|
||||
|
||||
__tablename__ = "plex_states"
|
||||
__table_args__ = (UniqueConstraint("user_id", "item_id", name="uq_plex_user_item"),)
|
||||
@@ -1556,7 +1496,7 @@ class PlexPlaylistItem(Base):
|
||||
|
||||
|
||||
class ComfortPin(Base, TimestampMixin):
|
||||
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X" (E24).
|
||||
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X".
|
||||
|
||||
Deliberately NOT a boolean on `VideoState`/`PlexState`, and NOT a polymorphic `(kind, ref)` string
|
||||
pair. A flag cannot ORDER a shelf and cannot hold a channel or a show — and comfort media is
|
||||
|
||||
@@ -25,7 +25,7 @@ def norm_lang(lang: str | None) -> str:
|
||||
return "hu" if lang == "hu" else "en"
|
||||
|
||||
|
||||
# Every card speaks the reader's language (E30): `lang="hu"` serves the HU overlay twin's title and
|
||||
# Every card speaks the reader's language: `lang="hu"` serves the HU overlay twin's title and
|
||||
# summary, falling back to EN where a row has none — the same rule as browse.unified_library's
|
||||
# `title_col`/`summary_col`. These rails once served English to a Hungarian reader.
|
||||
def display_title(row: PlexItem | PlexShow, lang: str) -> str:
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Server-side ONLY: uses the admin ``X-Plex-Token`` from sysconfig. Playback never goes through
|
||||
this client — Siftlode plays the LOCAL physical file (see app.plex.paths). Plex is used purely for
|
||||
metadata, image proxying, and (P5) optional watch-state write-back. Requests JSON via the
|
||||
metadata, image proxying, and optional watch-state write-back. Requests JSON via the
|
||||
``Accept: application/json`` header (Plex serves XML by default).
|
||||
"""
|
||||
|
||||
@@ -65,7 +65,7 @@ class PlexClient:
|
||||
except ValueError as e:
|
||||
raise PlexError("Plex returned a non-JSON response") from e
|
||||
|
||||
# --- Connectivity + sections (P0) ---
|
||||
# --- Connectivity + sections ---
|
||||
def server_info(self) -> dict:
|
||||
"""Root identity (friendlyName / version) — used by the admin 'Test connection'."""
|
||||
return self._get("/")
|
||||
@@ -75,7 +75,7 @@ class PlexClient:
|
||||
mc = self._get("/library/sections")
|
||||
return mc.get("Directory", []) or []
|
||||
|
||||
# --- Catalog mirror (P1) ---
|
||||
# --- Catalog mirror ---
|
||||
def section_items(
|
||||
self, key: str, item_type: int, start: int = 0, size: int = 200
|
||||
) -> tuple[list[dict], int | None]:
|
||||
@@ -88,14 +88,14 @@ class PlexClient:
|
||||
"type": item_type,
|
||||
# includeGuids=1 carries each item's Guid array (tmdb/imdb/tvdb ids) in the cheap
|
||||
# listing, so the sync can capture the canonical guid without a per-item rich fetch —
|
||||
# the pairing key for the HU overlay (E5 S5).
|
||||
# the pairing key for the HU overlay.
|
||||
"includeGuids": 1,
|
||||
"X-Plex-Container-Start": start,
|
||||
"X-Plex-Container-Size": size,
|
||||
},
|
||||
)
|
||||
# `totalSize` only: `size` is THIS page's count, and read as the total it made a single
|
||||
# page look like the whole section — "complete" to a prune or a reconcile (E32 review).
|
||||
# page look like the whole section — "complete" to a prune or a reconcile.
|
||||
total = mc.get("totalSize")
|
||||
return mc.get("Metadata", []) or [], None if total is None else int(total)
|
||||
|
||||
@@ -137,12 +137,12 @@ class PlexClient:
|
||||
mc = self._get(f"/library/collections/{rating_key}/children")
|
||||
return mc.get("Metadata", []) or []
|
||||
|
||||
# --- Collection editing (P2 — write-back to Plex; all shared/global, admin-gated in the routes) ---
|
||||
# --- Collection editing (write-back to Plex; all shared/global, admin-gated in the routes) ---
|
||||
@property
|
||||
def machine_id(self) -> str:
|
||||
"""Server machineIdentifier — needed in the `uri` of collection add/create calls. Cached.
|
||||
Without one the uri would read `server:///…`, a malformed write Plex answers in its own way;
|
||||
it is refused here, by name (E32 S2)."""
|
||||
it is refused here, by name."""
|
||||
if not getattr(self, "_machine_id", None):
|
||||
mid = self._get("/identity").get("machineIdentifier")
|
||||
if not mid:
|
||||
@@ -206,7 +206,7 @@ class PlexClient:
|
||||
items = mc.get("Metadata", []) or []
|
||||
return items[0] if items else None
|
||||
|
||||
# --- Watch-state write-back (P5 Phase B — Siftlode→Plex push; acts on the token's account) ---
|
||||
# --- Watch-state write-back (Siftlode→Plex push; acts on the token's account) ---
|
||||
_LIBRARY_IDENTIFIER = "com.plexapp.plugins.library"
|
||||
|
||||
def scrobble(self, rating_key: str) -> None:
|
||||
@@ -243,7 +243,7 @@ class PlexClient:
|
||||
f"&state={state}&time={int(time_ms)}&duration={int(duration_ms)}",
|
||||
)
|
||||
|
||||
# --- Watch-state read for incremental sync (P5 Phase C) ---
|
||||
# --- Watch-state read for incremental sync ---
|
||||
def accounts(self) -> list[dict]:
|
||||
"""Server accounts (the owner + any managed/home users): id, name. Used to resolve the
|
||||
owner's accountID so the shared watch-history feed can be filtered to just their views."""
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Parse the "info page" rich metadata out of a full Plex metadata dict, and persist the subset that
|
||||
the detail endpoints serve from the DB row (E5 S5b).
|
||||
the detail endpoints serve from the DB row.
|
||||
|
||||
Shared by the live info-page path (routes.plex.browse) and the background rich re-sync (plex.sync) so
|
||||
the two never diverge — the same one-parser rule that keeps `people.py`'s cast handling consistent.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""One Hungarian word per Plex genre (E30).
|
||||
"""One Hungarian word per Plex genre.
|
||||
|
||||
Plex's Hungarian agent returns a MIXED vocabulary: some genres come back translated ("Dráma",
|
||||
"Vígjáték"), others stay English ("Action", "Adventure", "Biography"), and a title with no HU twin has
|
||||
|
||||
@@ -26,7 +26,7 @@ _CT_EXT = {ct: ext for ext, ct in _EXT_CT}
|
||||
# Target box per variant — Plex resizes to these server-side. The full poster (400px) covers a detail
|
||||
# page's ≤224px poster at 2× DPR, the full art (1280px) a hero backdrop. A card is much smaller: a grid
|
||||
# poster is ~144px and a shelf's landscape art ~212px, so the `_sm` pair serves those at 1.33× DPR
|
||||
# (E31 S4e: cards fetched the 400 poster, 42 KB, and the 1280 art for a 212px picture). The browser
|
||||
# (cards used to fetch the 400 poster, 42 KB, and the 1280 art for a 212px picture). The browser
|
||||
# picks between a pair itself (`srcset`, built by the frontend's `lib/plexImage` cardImage, whose
|
||||
# widths plexImage.contract.test holds to these).
|
||||
SIZES = {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""The HU overlay: a Hungarian Plex section written onto its English twin's rows (E30), split out of
|
||||
`plex/sync.py` (E32 S2), which runs it once per paired section during the full sync.
|
||||
"""The HU overlay: a Hungarian Plex section written onto its English twin's rows, split out of
|
||||
`plex/sync.py`, which runs it once per paired section during the full sync.
|
||||
|
||||
The pairing is admin-configured (`plex_overlay_map`, ``huKey:enKey,...``). An overlay section creates
|
||||
no browsable rows of its own; it only fills the `*_hu` columns of the EN rows it matches by guid.
|
||||
@@ -41,7 +41,7 @@ def sync_overlay(
|
||||
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/tagline/genres
|
||||
differ by language; the language-neutral people_text feeds both search vectors. A show section's
|
||||
episodes are overlaid too (E30): the HU section lists every episode with its Hungarian title and
|
||||
episodes are overlaid too: the HU section lists every episode with its Hungarian title and
|
||||
summary, and the episode rows live in the EN show library.
|
||||
|
||||
This is a full RECONCILE: after a COMPLETE HU listing every guid-bearing EN row's HU fields are
|
||||
@@ -141,7 +141,7 @@ def _clear_hu(by_guid: dict, episodes: bool) -> None:
|
||||
cheap. The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking
|
||||
one, from its own English list — otherwise the HU genre list mixed "Drama" (no twin) with "Dráma".
|
||||
Only after a COMPLETE listing: one a Plex hiccup cut short would clear the HU text of every title
|
||||
it failed to reach, until the next full sync (E32 S2, the watch reconcile's twin)."""
|
||||
it failed to reach, until the next full sync (the watch reconcile's twin)."""
|
||||
for group in by_guid.values():
|
||||
for row in group:
|
||||
row.title_hu = row.summary_hu = None
|
||||
|
||||
@@ -160,7 +160,7 @@ def people_by_credits(
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
# --- Portraits (E30 S6) ---------------------------------------------------------------------------
|
||||
# --- Portraits ---------------------------------------------------------------------------
|
||||
# A Plex CDN headshot is a full-size original (~170 KB on average, measured). Plex's photo transcoder
|
||||
# fits it into this box (~13 KB), which covers the largest place a portrait is drawn (the person page's
|
||||
# 6rem avatar) at 2x density and a raised text size. The box is part of the cache key, like a poster's:
|
||||
@@ -226,7 +226,7 @@ _DEAD = frozenset({404, 410})
|
||||
|
||||
|
||||
def warm_portraits(db: Session, limit: int) -> dict:
|
||||
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first (E30 S6), so a
|
||||
"""Cache up to `limit` not-yet-cached headshots, the most-credited people first, so a
|
||||
person's face comes from local disk. Needs Plex (the resize) and the Plex module on; without them
|
||||
nothing is prefetched — full-size originals for every person would cost gigabytes. A headshot Plex
|
||||
answers "not found" for is marked and left for PORTRAIT_RETRY; Plex failing in a row ends the run."""
|
||||
|
||||
+10
-10
@@ -31,9 +31,9 @@ PLEX_SORT_KEYS: tuple[str, ...] = (
|
||||
"title",
|
||||
)
|
||||
|
||||
# --- Language-aware column pickers (E5 S6) ----------------------------------
|
||||
# --- Language-aware column pickers ----------------------------------
|
||||
#
|
||||
# The HU-overlay twin (E5 S5) stores Hungarian metadata in `*_hu` columns on the SAME EN row. When
|
||||
# The HU-overlay twin stores Hungarian metadata in `*_hu` columns on the SAME EN row. When
|
||||
# the app language is `hu` we serve those, falling back to the EN column where a row has no HU twin.
|
||||
# Two flavours of picker, on purpose:
|
||||
# * DISPLAY / FILTER (`title_col`/`genres_col`): `coalesce(*_hu, *)` — an EN fallback so a row with
|
||||
@@ -41,8 +41,8 @@ PLEX_SORT_KEYS: tuple[str, ...] = (
|
||||
# HU summary directly off the loaded row, so no `summary_col` SQL helper is needed.)
|
||||
# * SEARCH (`vector_col` + `field_recheck`): the raw `*_hu` column coalesced to '' — MIRRORING the
|
||||
# generated `search_vector_hu` definition (which uses `coalesce(title_hu, '')`, NOT an EN
|
||||
# fallback), so the field recheck stays a strict subset of the GIN vector prefilter (the S2
|
||||
# invariant). `people_text` is language-neutral and shared by both vectors.
|
||||
# fallback), so the field recheck stays a strict subset of the GIN vector prefilter (the
|
||||
# field-search invariant). `people_text` is language-neutral and shared by both vectors.
|
||||
|
||||
|
||||
def title_col(model, lang: str):
|
||||
@@ -216,7 +216,7 @@ def meta_filter_conds(model, p: dict, lang: str = "en") -> list:
|
||||
stds = split_csv(p.get("studios"))
|
||||
if stds: # OR
|
||||
conds.append(model.studio.in_(stds))
|
||||
for kw in split_csv(p.get("keywords")): # AND — TMDB keyword chips from the detail page (E5 S5)
|
||||
for kw in split_csv(p.get("keywords")): # AND — TMDB keyword chips from the detail page
|
||||
conds.append(model.keywords.contains([kw]))
|
||||
csel = split_csv(p.get("collections"))
|
||||
if (
|
||||
@@ -232,7 +232,7 @@ def apply_meta_filters(q, model, p: dict, lang: str = "en"):
|
||||
return q.where(*conds) if conds else q
|
||||
|
||||
|
||||
# --- Playback-class filter (E7) — MOVIE-ONLY (a show is an aggregate with no single class) ---------
|
||||
# --- Playback-class filter — MOVIE-ONLY (a show is an aggregate with no single class) ---------
|
||||
# The 4 classes the feed filter + facet expose, derived from the stored `playable` + `transcode_hw`.
|
||||
# ONE definition (this SQL CASE) so the filter WHERE and the facet counts can't drift: `playable_conds`
|
||||
# filters on it, `facets` groups by it. transcode_hw IS NOT True groups NULL (unprobed) with software.
|
||||
@@ -278,7 +278,7 @@ class LibraryFilters:
|
||||
actors: str | None = None
|
||||
actor_mode: str = "any" # how multiple actors combine: any (OR, default) | all (AND)
|
||||
studios: str | None = None
|
||||
keywords: str | None = None # TMDB keyword chips (E5 S5), CSV; language-neutral
|
||||
keywords: str | None = None # TMDB keyword chips, CSV; language-neutral
|
||||
collections: str | None = None # CSV of collection rating_keys
|
||||
collection_mode: str = "any" # how multiple collections combine: any (OR, default) | all (AND)
|
||||
playable: str | None = None # CSV of playback classes: direct|remux|transcode_hw|transcode_sw
|
||||
@@ -311,7 +311,7 @@ class LibraryFilters:
|
||||
# watch state. The SAME predicate has to be applied in two places — `unified_library` (the grid) and
|
||||
# `facets.wsq` (so the facet counts reflect exactly the visible set) — and they MUST stay identical:
|
||||
# a hidden movie kept out of the grid must not inflate the facets either. These helpers are the one
|
||||
# home for that predicate so the two call sites can't drift (E3 S1; pinned by test_plex_facets_grid_
|
||||
# home for that predicate so the two call sites can't drift (pinned by test_plex_facets_grid_
|
||||
# parity). Callers pass the already-joined state expressions, so this module stays model-agnostic:
|
||||
# * movies — `st` is an aliased PlexState outer-joined on (item_id, user_id).
|
||||
# * shows — `tot`/`wat`/`inp` are the coalesced totals from `rollup.agg_subquery`.
|
||||
@@ -367,9 +367,9 @@ def apply_show_state_filter(query, tot, wat, inp, show: str):
|
||||
def content_libs(db: Session) -> tuple[list[int], list[int]]:
|
||||
"""The BROWSABLE content libraries, partitioned into (movie_lib_ids, show_lib_ids) — the single
|
||||
source of the browse scope. Excludes HU overlay libraries (`overlay_of` set): those are never
|
||||
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows (E5 S5).
|
||||
browsed on their own, they only lend their Hungarian metadata to the EN twin's rows.
|
||||
|
||||
Lives here, in the query layer, rather than in `routes/plex/_shared.py` where it started: E24's
|
||||
Lives here, in the query layer, rather than in `routes/plex/_shared.py` where it started: the
|
||||
Home surface needs the same scope, and a domain package importing from `routes/` would invert the
|
||||
one-way dependency CLAUDE.md draws. It never touched request state, so nothing was lost by the
|
||||
move."""
|
||||
|
||||
@@ -13,10 +13,10 @@ from app.models import PlexItem, PlexState
|
||||
# Sort sentinel for a rail row whose timestamp is missing, so `key=lambda r: r[0] or NO_TS` reads the
|
||||
# same at every rail that merges two type-homogeneous lists in Python.
|
||||
#
|
||||
# It is NOT a crash fix, whatever the E24 commit that introduced it says. The `(ts is not None, ts)`
|
||||
# It is NOT a crash fix, whatever the commit that introduced it says. The `(ts is not None, ts)`
|
||||
# key it replaced cannot raise: tuple comparison finds the first UNEQUAL element via `==`, so two
|
||||
# `(False, None)` keys compare equal and `None < None` is never reached — verified by execution
|
||||
# during the E24 epic-end review, over every permutation of null/non-null rows. What the sentinel
|
||||
# during a review, over every permutation of null/non-null rows. What the sentinel
|
||||
# buys is one idiom instead of two, and a key that says "missing sorts last" outright rather than
|
||||
# encoding it in a boolean flag. A BARE `r[0]` would genuinely raise, which is the mutation the
|
||||
# rails' tests do guard against.
|
||||
|
||||
+40
-21
@@ -15,7 +15,7 @@ HLS muxer does the segmentation (the only reliable way to cut a stream-copy at k
|
||||
|
||||
Sessions live in this (single) API process; a periodic reaper kills idle ones and frees the temp
|
||||
segments. The remux is CPU-light (video copy + a tiny audio transcode), so it runs fine on the
|
||||
CPU-only prod host; only P3's full transcode is CPU-heavy.
|
||||
CPU-only prod host; only the full transcode is CPU-heavy.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -120,8 +120,8 @@ class HlsSession:
|
||||
# Unguessable per-session id, carried in the URLs the client is handed. A seek-restart
|
||||
# replaces the viewer's registered session for the item — and the old player's next request
|
||||
# for `seg_12.ts`, resolved by (viewer, rating_key) alone, was answered 200 with the NEW session's twelfth segment, produced from a different offset
|
||||
# under `-copyts`. Wrong bytes, not an error, so the client's reclaim never fired on it
|
||||
# (deep-review round 5). A request naming a session that is no longer registered is a 404,
|
||||
# under `-copyts`. Wrong bytes, not an error, so the client's reclaim never fired on it.
|
||||
# A request naming a session that is no longer registered is a 404,
|
||||
# which the player recovers from by restarting at the playhead.
|
||||
self.id = secrets.token_urlsafe(9)
|
||||
self.key = key # the item's rating_key (what the logs name)
|
||||
@@ -413,7 +413,7 @@ def _venc_opts(
|
||||
def _render_node_present() -> bool:
|
||||
"""Whether the VAAPI render node is available — i.e. `/dev/dri` was passed into the container.
|
||||
Present on prod after the iGPU enablement; absent on the CPU-only dev box and in the localdev
|
||||
Docker. Its own function so the selection is monkeypatchable in tests and one place for S1c's knob."""
|
||||
Docker. Its own function so the selection is monkeypatchable in tests and the one place to change it."""
|
||||
return Path(_VAAPI_DEVICE).exists()
|
||||
|
||||
|
||||
@@ -576,7 +576,7 @@ def _ffmpeg_cmd(
|
||||
entry += ",default:yes"
|
||||
var.append(entry)
|
||||
# The name patterns come from the vocabulary functions, with ffmpeg's own placeholders passed
|
||||
# through them — so the producer and the parser cannot spell the layout differently (round 7).
|
||||
# through them — so the producer and the parser cannot spell the layout differently.
|
||||
args += _HLS_TAIL + [
|
||||
"-master_pl_name",
|
||||
_MASTER_PL,
|
||||
@@ -744,8 +744,9 @@ def start_session(
|
||||
# `replace_only` makes the supersede identity-aware for the runtime fallback: the whole
|
||||
# check-and-replace happens under ONE _lock, so a newer same-key session (a concurrent
|
||||
# seek-restart) installed between the fallback's decision and here is never clobbered. When the
|
||||
# registered session is no longer `replace_only`, we abort and return None (the caller yields to
|
||||
# whoever is registered now). The INITIAL spawn never gets None — see its call site.
|
||||
# registered session is no longer `replace_only`, we abort and return None, and the caller
|
||||
# answers with its own dead session — never the newer one, whose id a stale requester would
|
||||
# stop. The INITIAL spawn never gets None — see its call site.
|
||||
# Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and
|
||||
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
||||
preset, crf, max_height = "veryfast", 20, 0
|
||||
@@ -840,12 +841,13 @@ def start_session(
|
||||
)
|
||||
respawned = _spawn(False, False, replace_only=s)
|
||||
if respawned is None:
|
||||
# Superseded/culled mid-fallback — yield whoever is registered now (a newer seek-restart), or
|
||||
# our own now-dead session if the key was cap-culled. Never None here: the POST /session route
|
||||
# turns a None return into a "Media file not found on disk" 404 — dishonest for a file that
|
||||
# exists. A (dead) session instead degrades to a retryable 404 ("No active playback session" /
|
||||
# "Not available yet") the client re-requests.
|
||||
return current_session(user_id, key) or s
|
||||
# Superseded/culled mid-fallback — answer with our own now-dead session. Never the newer one
|
||||
# registered meanwhile: this answer goes to the request that LOST, whose player drops it as
|
||||
# stale and stops the session it names (`stop_session`), which stopped the playback that won.
|
||||
# Never None either: the POST /session route turns a None return into a "Media file not found
|
||||
# on disk" 404 — dishonest for a file that exists. A dead session instead degrades to a
|
||||
# retryable 404 ("No active playback session" / "Not available yet") the client re-requests.
|
||||
return s
|
||||
s = respawned
|
||||
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
||||
if ready:
|
||||
@@ -887,7 +889,7 @@ def current_session(user_id: int, key: str) -> HlsSession | None:
|
||||
# The names ffmpeg writes into a session directory, in ONE place — and "one place" means the argv
|
||||
# BUILDS its patterns from these same functions (`segment_name("%d", "%v")`), not that a comment
|
||||
# says so. Getting this wrong is invisible in both directions: a name we failed to anticipate used
|
||||
# to fall back to "wait and see" (round 6's HIGH), and now that the rule is positive it falls back
|
||||
# to fall back to "wait and see", and now that the rule is positive it falls back
|
||||
# to a 404 instead — so the producer and the parser have to be literally the same spelling.
|
||||
#
|
||||
# MEASURED (ffmpeg 7.1.5 in the API container — the binary that actually writes these files — and
|
||||
@@ -901,7 +903,7 @@ def current_session(user_id: int, key: str) -> HlsSession | None:
|
||||
# SESSION and not to HLS in general: a single-audio session can never write `master.m3u8`,
|
||||
# `stream_*.m3u8` or any `seg_<variant>_<n>.ts`, and a multi-audio one can never write `index.m3u8`
|
||||
# or `seg_<n>.ts`. Accepting the union meant a hopeless name of the other shape was waited out for
|
||||
# the full playlist/segment timeout — the threadpool parking this guard exists to end (round 7).
|
||||
# the full playlist/segment timeout — the threadpool parking this guard exists to end.
|
||||
_MASTER_PL = "master.m3u8"
|
||||
_INDEX_PL = "index.m3u8"
|
||||
_SEG_NAME_RE = re.compile(r"^seg_(?:([0-9a-z]{1,4})_)?(\d{1,9})\.ts$")
|
||||
@@ -953,12 +955,12 @@ def artifact_may_arrive(s: "HlsSession", filename: str) -> bool:
|
||||
name belonging to the other shape is as hopeless here as a random string. Reading those as "an
|
||||
HLS name, so wait and see" left the whole class reachable — measured through the real route
|
||||
against a live single-audio session, `master.m3u8` and `stream_q7z.m3u8` were 404s after 30.0 s
|
||||
and `seg_a0_0.ts` / `seg_zzzz_0.ts` after 20.1 s each (deep-review round 7).
|
||||
and `seg_a0_0.ts` / `seg_zzzz_0.ts` after 20.1 s each.
|
||||
|
||||
Saying no at once matters because the route that waits is a SYNC one: it holds a thread off the
|
||||
~40-slot pool, and a pooled DB connection with it, for the whole timeout. An ordinary seek past
|
||||
the produced region was enough to park one for 20 s, and forty concurrent requests for a name
|
||||
that will never exist parked the whole API (deep-review round 5). Round 5's guard read only
|
||||
that will never exist parked the whole API. An earlier guard read only
|
||||
single-audio segment names, so both halves of that were still fully reachable: every multi-audio
|
||||
name fell through, and so did any `.m3u8`/`.vtt` name at all."""
|
||||
if filename in playlist_names(s.variants):
|
||||
@@ -1093,15 +1095,15 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int:
|
||||
A finished session is also what a PAUSED viewer has: a remux runs far faster than realtime, so
|
||||
ffmpeg is done a minute or two in and writes ENDLIST; hls.js then stops asking for the playlist,
|
||||
and with its forward buffer full a paused player fetches nothing at all. With the 30 s rule that
|
||||
killed a one-minute pause, and playback died ~30 s after resuming (deep-review round 4).
|
||||
killed a one-minute pause, and playback died ~30 s after resuming.
|
||||
|
||||
What makes the short rule safe again is that fetching is no longer the only signal: a player
|
||||
parked MID-ITEM checkpoints its position every 10 s, and that route calls `touch_session`, so a
|
||||
viewer who is merely paused says so far more often than this rule asks (round 5). The heartbeat
|
||||
viewer who is merely paused says so far more often than this rule asks. The heartbeat
|
||||
stops once the item is finished or before a position is known (`usePlexProgress.flush`) — both
|
||||
states where reaping is correct, and where the player's 404 → reclaim recovers a viewer who
|
||||
scrubs back in anyway. Do not restate this as "every 10 s whether or not it is playing": round 6
|
||||
found that claim in two comments and not in the code. Round 4's blanket
|
||||
scrubs back in anyway. Do not restate this as "every 10 s whether or not it is playing":
|
||||
a review found that claim in two comments and not in the code. An earlier blanket
|
||||
`reap_abandoned` bought the same safety by keeping every finished session for ten minutes — each
|
||||
one a whole segmented copy of the film on a volume shared with the download center."""
|
||||
now = time.time()
|
||||
@@ -1117,6 +1119,23 @@ def reap_idle(finished_idle_s: float = _FINISHED_IDLE_S) -> int:
|
||||
return dropped
|
||||
|
||||
|
||||
def stop_session(user_id: int, key: str, session_id: str) -> bool:
|
||||
"""Stop this viewer's session for `key` now, because the player that asked for it is gone (left
|
||||
the item, closed, unloaded the page). Left to `reap_idle`, a transcode ran on to the end of the
|
||||
film first — measured: ~4 cores for a minute after leaving a resumed 480p film.
|
||||
|
||||
Only while `session_id` is still the registered one: the player names the session it played, and
|
||||
a seek-restart or a second tab may have replaced it meanwhile — that newer session is somebody's
|
||||
current playback, not this request's to end."""
|
||||
with _lock:
|
||||
s = _sessions.get((user_id, key))
|
||||
if s is None or s.id != session_id:
|
||||
return False
|
||||
_kill(s)
|
||||
_sessions.pop((user_id, key), None)
|
||||
return True
|
||||
|
||||
|
||||
def touch_session(user_id: int, key: str) -> None:
|
||||
"""Say this viewer is still there for `key`, without fetching anything. `last_access` used to move
|
||||
only on an HLS file fetch, which a paused player does not do — so the only way to keep a paused
|
||||
|
||||
@@ -12,7 +12,7 @@ on a NON-empty listing — a section that returns zero items is left untouched,
|
||||
glitch that yields an empty page can't wipe a library (the metadata is cheap to re-sync; an accidental
|
||||
mass-delete is the only real risk). Library-level reconcile (dropping a whole removed section, the
|
||||
dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are fetched lazily on the
|
||||
item-detail/player path (P2), not here.
|
||||
item-detail/player path, not here.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
@@ -150,7 +150,7 @@ _GUID_PREFERENCE = ("tmdb", "imdb", "tvdb")
|
||||
|
||||
|
||||
def _canonical_guid(meta: dict) -> str | None:
|
||||
"""The canonical external id used to pair an EN title with its HU-overlay twin (E5 S5). Reads the
|
||||
"""The canonical external id used to pair an EN title with its HU-overlay twin. Reads the
|
||||
`Guid` array (present only with includeGuids=1 in the listing), preferring tmdb > imdb > tvdb,
|
||||
normalised to '<provider>:<id>' (e.g. 'tmdb:12345'). None when no known provider id is present."""
|
||||
found: dict[str, str] = {}
|
||||
@@ -419,7 +419,7 @@ def _paginate(plex: PlexClient, key: str, item_type: int, progress: dict | None
|
||||
# A truthy `total` we reached = a trustworthy complete listing. `total` unknown/zero-with-rows
|
||||
# (a Plex that doesn't report totalSize) is treated as NOT complete, so prune stays conservative.
|
||||
# `total` itself is kept for a caller to whom a server that SAID 0 is news it can trust (the
|
||||
# watch reconcile); an empty mirror or overlay must not read it so (E32 S2).
|
||||
# watch reconcile); an empty mirror or overlay must not read it so.
|
||||
progress["complete"] = bool(total) and start >= total
|
||||
progress["total"] = total
|
||||
|
||||
@@ -447,7 +447,7 @@ def _apply_facet_fields(row, meta: dict) -> None:
|
||||
# full cast (up to 24, grown by the rich backfill/info-page enrich), silently breaking the
|
||||
# person-page cast filter for names past the top ~20. The watermark gate below (`people_enriched_at
|
||||
# is None`) normally protects this, but it is NOT enough on its own: a migration that resets the
|
||||
# watermark re-opens the regression (0065 nulled it, which is exactly how the S5b fix got undone in
|
||||
# watermark re-opens the regression (0065 nulled it, which is exactly how a fix got undone in
|
||||
# prod). So guard STRUCTURALLY too, by LENGTH against the listing cap: a stored cast LONGER than the
|
||||
# cap can only be an enriched fuller list, so keep it (robust to any watermark reset). A stored cast
|
||||
# at or below the cap is itself listing-sourced, so take the fresh listing — which tracks Plex cast
|
||||
@@ -694,7 +694,7 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
||||
def _believed_children(plex: PlexClient, rk: str, meta: dict) -> list[dict] | None:
|
||||
"""A collection's members, or None when the answer is not believed: no members from a collection
|
||||
whose own listing counts some (`childCount`) is a glitch, not a collection emptied — the same
|
||||
empty-answer rule as the collection listing's (E32 review). Rebuilding on it showed the
|
||||
empty-answer rule as the collection listing's. Rebuilding on it showed the
|
||||
collection empty until the next sync."""
|
||||
children = plex.collection_children(rk)
|
||||
count = str(meta.get("childCount") or "0")
|
||||
@@ -745,7 +745,7 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
||||
if not cols and existing:
|
||||
# The catalog's own rule (see the module doc): an empty listing while we hold rows is a Plex
|
||||
# glitch, not a library emptied of collections. Reconciling on it dropped every collection
|
||||
# (its user-set `editable` flag with it) and every member's keys (E32 S2).
|
||||
# (its user-set `editable` flag with it) and every member's keys.
|
||||
log.warning(
|
||||
"Plex listed no collections for a library that has some; keeping them this run",
|
||||
extra=logs.event("plex.sync.collections_empty", library=lib.plex_key),
|
||||
|
||||
+11
-11
@@ -1,4 +1,4 @@
|
||||
"""TMDB enrichment for the mirrored Plex catalog (E5 S2).
|
||||
"""TMDB enrichment for the mirrored Plex catalog.
|
||||
|
||||
A thin synchronous TMDB v3 client plus the parse/persist of the enrichment fields onto plex_items /
|
||||
plex_shows, matched by the stored canonical `guid` ("tmdb:12345", or imdb/tvdb resolved via /find).
|
||||
@@ -134,7 +134,7 @@ class TmdbClient:
|
||||
return cast("list[dict] | None", self._get("/configuration/countries"))
|
||||
|
||||
def fetch_person(self, person_id: int) -> dict | None:
|
||||
"""One person's bio + combined credits (E5 S6) + the bio's translations (E30 S4), in a single
|
||||
"""One person's bio + combined credits + the bio's translations, in a single
|
||||
append_to_response request."""
|
||||
return self._get(
|
||||
f"/person/{person_id}", {"append_to_response": "combined_credits,translations"}
|
||||
@@ -226,7 +226,7 @@ def parse_tmdb(payload: dict, media_type: str) -> dict:
|
||||
crew.sort(key=lambda c: priority.get(c["job"], 99))
|
||||
crew = crew[:12]
|
||||
|
||||
# name -> tmdb person id, for the plex_people foundation (S6 fetches bios off these ids).
|
||||
# name -> tmdb person id, for the plex_people foundation (bios are fetched off these ids).
|
||||
people: list[dict] = []
|
||||
for c in (credits.get("cast") or [])[:30]:
|
||||
if c.get("name") and c.get("id"):
|
||||
@@ -266,7 +266,7 @@ def parse_tmdb(payload: dict, media_type: str) -> dict:
|
||||
c.get("name") for c in (payload.get("production_companies") or []) if c.get("name")
|
||||
][:8]
|
||||
or None,
|
||||
# The ISO code beside TMDB's English name (E30): the client names the country in the reader's
|
||||
# The ISO code beside TMDB's English name: the client names the country in the reader's
|
||||
# language from the code; the name is the fallback for a code it cannot name.
|
||||
"production_countries": [
|
||||
{"code": c.get("iso_3166_1") or None, "name": c["name"]}
|
||||
@@ -373,14 +373,14 @@ def enrich_batch(db: Session, tmdb: TmdbClient, limit: int) -> int:
|
||||
|
||||
|
||||
def _country(c: dict | str) -> dict:
|
||||
"""One stored production country as served: `{code, name}`. A row enriched before E30 stored the
|
||||
"""One stored production country as served: `{code, name}`. An older row stored the
|
||||
bare English name; until `backfill_country_codes` reaches it, it serves with no code (the client
|
||||
shows the name)."""
|
||||
return c if isinstance(c, dict) else {"code": None, "name": c}
|
||||
|
||||
|
||||
def backfill_country_codes(db: Session, tmdb: TmdbClient) -> int:
|
||||
"""Give the countries stored before E30 (bare English names) their ISO codes, from TMDB's own
|
||||
"""Give the older countries (bare English names) their ISO codes, from TMDB's own
|
||||
country list — the names came from that list, so they match it exactly. One request, and only
|
||||
while such a row is left: re-fetching each title instead would take the enrichment batch days.
|
||||
A name the list does not hold keeps no code. Returns the number of rows rewritten."""
|
||||
@@ -418,7 +418,7 @@ def run_enrich(db: Session) -> dict:
|
||||
|
||||
|
||||
def run_people_enrich(db: Session) -> dict:
|
||||
"""Scheduler entry point of people_enrich (E30 S6): a batch of person bios (the most-credited
|
||||
"""Scheduler entry point of people_enrich: a batch of person bios (the most-credited
|
||||
first) and a batch of portraits. Each half is a no-op — never an error — without its TMDB key /
|
||||
without Plex."""
|
||||
batch = settings.people_enrich_batch
|
||||
@@ -475,7 +475,7 @@ def resolve_recs(db: Session, recs: list[dict] | None, *, lang: str) -> list[dic
|
||||
"""Tag each recommendation with whether it's in OUR library (matched by guid `tmdb:<id>`, bucketed
|
||||
by media_type — see resolve_library_guids) and, if so, its rating_key — so the client can deep-link
|
||||
the ones we have and merely display the rest. One we have is named as the library names it, in
|
||||
`lang` (E30); one we lack keeps the title TMDB gave."""
|
||||
`lang`; one we lack keeps the title TMDB gave."""
|
||||
if not recs:
|
||||
return []
|
||||
movies, shows = resolve_library_guids(db, recs)
|
||||
@@ -515,7 +515,7 @@ def serve_block(db: Session, row, *, lang: str) -> dict | None:
|
||||
}
|
||||
|
||||
|
||||
# --- Person enrichment (E5 S6) -------------------------------------------------------------------
|
||||
# --- Person enrichment -------------------------------------------------------------------
|
||||
# A person's bio + birth facts + a "Known for" credit pool, fetched once — by the people_enrich job, or
|
||||
# by the first person-page open if the job has not reached them yet — and served from the plex_people row. `known_for` stores the top combined-credits ids; the person
|
||||
# endpoint resolves them to library rows by guid at serve time (like recommendations).
|
||||
@@ -525,7 +525,7 @@ def serve_block(db: Session, row, *, lang: str) -> dict | None:
|
||||
# credits are often not the 15 a given library holds.
|
||||
_KNOWN_FOR_MAX = 40
|
||||
# TMDB genre ids of the shows a person visits as a guest (News, Reality, Talk). Ranked by popularity,
|
||||
# talk-show visits crowded out the films (Tom Cruise: 14 of 15 were talk shows; E30 S6).
|
||||
# talk-show visits crowded out the films (Tom Cruise: 14 of 15 were talk shows).
|
||||
_GUEST_GENRES = {10763, 10764, 10767}
|
||||
# A credit in which the person plays themselves ("Self", "Himself - Guest", "Herself (archive footage)").
|
||||
_SELF_CREDIT = re.compile(r"^\s*(self|himself|herself|themselves)\b", re.IGNORECASE)
|
||||
@@ -646,7 +646,7 @@ _PEOPLE_FAILS_IN_A_ROW = 3
|
||||
|
||||
|
||||
def enrich_people_batch(db: Session, tmdb: TmdbClient, limit: int) -> dict:
|
||||
"""Fetch up to `limit` linked, not-yet-enriched people (E30 S6), the most-credited first, so a
|
||||
"""Fetch up to `limit` linked, not-yet-enriched people, the most-credited first, so a
|
||||
person page reads its bio from the row instead of waiting on TMDB. Each person is committed as it
|
||||
lands: a page open during the run already sees it. A failed lookup is skipped (left unstamped for the
|
||||
next run); _PEOPLE_FAILS_IN_A_ROW of them in a row end the batch."""
|
||||
|
||||
@@ -11,7 +11,7 @@ already pages) and upsert them into the owner's `plex_states`. Plex wins on over
|
||||
exist only in Siftlode are left untouched (union, Plex-authoritative on the intersection).
|
||||
|
||||
Phases B (Siftlode→Plex push via scrobble/timeline) and C (incremental Plex→Siftlode via the watch
|
||||
history + onDeck) build on this and land in later ships. As of S2 Plex is the source of truth: the
|
||||
history + onDeck) build on this and land in later ships. Plex is now the source of truth: the
|
||||
player pushes the resume position back to Plex on the periodic checkpoint (not just the final flush),
|
||||
and the incremental pull resolves a genuine divergence to Plex's state (see `_pull_apply`) rather than
|
||||
running a last-write-wins comparison.
|
||||
@@ -81,7 +81,7 @@ def _scan_plex_states(
|
||||
The settled keys matter to the reconcile alone: it reads "absent from the scan" as "Plex
|
||||
un-watched it", which a walk cut short by a Plex hiccup does not mean (see `_paginate`). Settled
|
||||
is decided PER SECTION: one section that never settles (a library emptied on Plex while its
|
||||
mirrored items are kept) must not stop un-watches syncing in all the others (E32 review)."""
|
||||
mirrored items are kept) must not stop un-watches syncing in all the others."""
|
||||
item_id_by_rk = _rk_to_id(db)
|
||||
wanted = _enabled_section_keys(db)
|
||||
# The sections we mirror something from: a total of 0 from one of those contradicts the mirror.
|
||||
@@ -117,7 +117,7 @@ def _scan_plex_states(
|
||||
out[item_id] = res
|
||||
# A section the server SAYS is empty (total 0, nothing listed) settles too. Believed only for
|
||||
# a section we mirror nothing from: one whose items we hold answering 0 is a Plex that lost
|
||||
# the library for a moment, not one that un-watched all of it (E32 review) — so it is left
|
||||
# the library for a moment, not one that un-watched all of it — so it is left
|
||||
# unsettled, and its rows are not cleared.
|
||||
listed = scanned > before
|
||||
said_empty = progress.get("total") == 0 and not listed and key not in mirrored
|
||||
@@ -402,7 +402,7 @@ def _pull_apply(
|
||||
if st is not None and not st.synced_to_plex:
|
||||
st.synced_to_plex = True # already matches Plex → just settle the flag, no churn
|
||||
return None
|
||||
# Plex-authoritative (S2): on a genuine divergence for an item Plex knows about, Plex wins — no
|
||||
# Plex-authoritative: on a genuine divergence for an item Plex knows about, Plex wins — no
|
||||
# last-write-wins tug-of-war. This is safe because prompt write-back (playback.py pushes the resume
|
||||
# position to Plex on the ~10s checkpoint, not just on the final flush) keeps Plex current, so a
|
||||
# local edit that's "newer" than Plex has almost always already been pushed and matched above by
|
||||
@@ -620,9 +620,8 @@ def _clear_unwatched(db: Session, user_id: int, plex_states: dict, settled: set[
|
||||
"""Clear the user's rows Plex no longer has a record for — the mirrored ones (Plex un-watched
|
||||
them); hidden and dirty rows stay (a dirty one is re-pushed by `_repush_dirty`). Only rows of the
|
||||
sections the scan SETTLED: a walk a Plex hiccup cut short leaves out rows it merely failed to
|
||||
reach, and clearing those un-watched them here until the next complete run (E32 S2); an item of
|
||||
a library the allowlist leaves out is absent from every scan, and was cleared on every run (E32
|
||||
review)."""
|
||||
reach, and clearing those un-watched them here until the next complete run; an item of
|
||||
a library the allowlist leaves out is absent from every scan, and was cleared on every run."""
|
||||
n = 0
|
||||
rows = db.execute(
|
||||
select(PlexState)
|
||||
|
||||
@@ -25,7 +25,7 @@ def report(current: int, total: int | None = None, phase: str | None = None) ->
|
||||
|
||||
|
||||
class JobFailed(RuntimeError):
|
||||
"""A job's own failure, said so the reader's language can name it (E31): the client reads
|
||||
"""A job's own failure, said so the reader's language can name it: the client reads
|
||||
`scheduler.failure.<reason>` and interpolates `params` (keep them to values the reader may see —
|
||||
numbers, ids, short codes). The message is the English line the log and `last_error` keep. An
|
||||
upstream's own error (a timeout, a refused connection) is not one of these: it stays as raised,
|
||||
|
||||
+18
-2
@@ -7,11 +7,13 @@ always be allowed; expensive backfill must yield to the remaining budget.
|
||||
|
||||
import contextlib
|
||||
import contextvars
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, cast
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.engine import CursorResult
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
@@ -96,6 +98,20 @@ def pacific_today():
|
||||
return datetime.now(_PACIFIC).date()
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Delete quota-history rows older than `retention_days` (<= 0 keeps them forever) and return how
|
||||
many went. Flushes only; the cleanup job commits. Today's rows are never in reach: the per-user
|
||||
search cap counts them, and the floor of two days outlasts any Pacific day (25 h at most)."""
|
||||
if retention_days <= 0:
|
||||
return 0
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=max(2, retention_days))
|
||||
# A DELETE answers with a CursorResult; `execute` is typed as the general Result, without rowcount.
|
||||
result = cast(
|
||||
CursorResult[Any], db.execute(delete(QuotaEvent).where(QuotaEvent.created_at < cutoff))
|
||||
)
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def pacific_day_start_utc() -> datetime:
|
||||
"""UTC instant of the current Pacific day's midnight — aligns 'today' with the budget reset."""
|
||||
start = datetime.now(_PACIFIC).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
"""In-process WebSocket connection registry for live server→client push.
|
||||
|
||||
Generic groundwork (notification module phase 2 is the first user): connections are kept in
|
||||
this process's memory, keyed by user_id, so a request handler can push an event to all of a
|
||||
user's open tabs/devices. Single-uvicorn-process assumption — if we ever run multiple workers,
|
||||
replace the in-memory map with a Redis pub/sub fan-out behind the same `push()` interface (same
|
||||
note as app.ratelimit).
|
||||
|
||||
`push()` is intentionally SYNC-callable: ordinary (sync) route handlers run in a threadpool, so
|
||||
they schedule the actual send onto the WebSocket event loop via run_coroutine_threadsafe. The
|
||||
connect/disconnect/_push coroutines all run on that one loop, so the connection map is only ever
|
||||
mutated from a single thread.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections import defaultdict
|
||||
|
||||
from starlette.websockets import WebSocket
|
||||
from app import logs
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
def __init__(self) -> None:
|
||||
self._conns: dict[int, set[WebSocket]] = defaultdict(set)
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
async def connect(self, user_id: int, ws: WebSocket) -> None:
|
||||
self._loop = asyncio.get_running_loop()
|
||||
await ws.accept()
|
||||
self._conns[user_id].add(ws)
|
||||
|
||||
async def disconnect(self, user_id: int, ws: WebSocket) -> None:
|
||||
conns = self._conns.get(user_id)
|
||||
if conns:
|
||||
conns.discard(ws)
|
||||
if not conns:
|
||||
self._conns.pop(user_id, None)
|
||||
|
||||
async def _push(self, user_id: int, payload: dict) -> None:
|
||||
for ws in list(self._conns.get(user_id, ())):
|
||||
try:
|
||||
await ws.send_json(payload)
|
||||
except Exception as exc: # noqa: BLE001 — one dead socket must not stop the others
|
||||
# Stale socket; it'll be cleaned up by its own disconnect handler. But this is
|
||||
# also where a live update goes missing: the UI simply never changes, and there
|
||||
# is no request behind it whose failure anyone could look up.
|
||||
log.debug(
|
||||
"a live update could not be pushed to a socket",
|
||||
extra=logs.event(
|
||||
"realtime.push_failed", user_id=user_id, reason=type(exc).__name__
|
||||
),
|
||||
)
|
||||
|
||||
def push(self, user_id: int, payload: dict) -> None:
|
||||
"""Schedule a push onto the WebSocket event loop. Safe to call from a sync route
|
||||
handler (threadpool). No-op if no WebSocket has ever connected (no loop captured)."""
|
||||
loop = self._loop
|
||||
if loop is None:
|
||||
return
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(self._push(user_id, payload), loop)
|
||||
except Exception: # noqa: BLE001 — a closed loop or socket must not fail the caller
|
||||
log.debug("ws push failed", exc_info=True)
|
||||
|
||||
|
||||
manager = ConnectionManager()
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Route-layer shared helpers for the API contract (R6 S1a).
|
||||
"""Route-layer shared helpers for the API contract.
|
||||
|
||||
Small, dependency-light guards that several route modules had hand-rolled with subtle
|
||||
drift. Keeping them here (imported by the route modules) means the "fetch a row or 404"
|
||||
@@ -9,16 +9,52 @@ one existence-leak policy.
|
||||
from typing import IO, Any, TypeVar
|
||||
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db import Base
|
||||
from app.models import User
|
||||
from app.models import POSITION_MAX_S, User
|
||||
from app.issues import gitea
|
||||
from app.refusal import refused
|
||||
from app.youtube.client import YouTubeError, refusal_reason
|
||||
|
||||
M = TypeVar("M", bound=Base)
|
||||
|
||||
# Resume-position thresholds of a player that only keeps a resume point (the YouTube player, the
|
||||
# library's): positions below this are "didn't really start" and within this of the end are
|
||||
# "basically finished" — neither is worth storing, so the position is cleared instead (no progress
|
||||
# bar). The client's local resume (`lib/resume.ts`) keeps the same two numbers, held equal by
|
||||
# `resume.contract.test.ts` beside it. NB: the Plex player intentionally uses a LARGER finish margin
|
||||
# (30s) because it marks long-form items watched, not just clears a resume bar — see
|
||||
# routes/plex/playback.py.
|
||||
PROGRESS_MIN_SECONDS = 5
|
||||
FINISH_MARGIN_SECONDS = 10
|
||||
|
||||
|
||||
class ProgressIn(BaseModel):
|
||||
"""A player's resume checkpoint (sent every few seconds while it plays, and as a beacon when the
|
||||
page goes away)."""
|
||||
|
||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||
# one into a 500.
|
||||
model_config = ConfigDict(allow_inf_nan=False)
|
||||
|
||||
# Accept a float and truncate (`kept_position`) — the player may post a raw currentTime (a float),
|
||||
# which the old `int(payload.get(...))` silently truncated. A plain `int` field would 422 that on
|
||||
# the progress hot path, losing the resume position; `float` keeps the old tolerance.
|
||||
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
||||
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
|
||||
|
||||
def kept_position(body: ProgressIn) -> int:
|
||||
"""The whole-second resume point worth storing for `body`, or 0 for "clear it" (too early to
|
||||
count as started, or close enough to the end to count as finished)."""
|
||||
position = max(0, int(body.position_seconds))
|
||||
duration = int(body.duration_seconds)
|
||||
near_end = duration > 0 and position > duration - FINISH_MARGIN_SECONDS
|
||||
return position if position >= PROGRESS_MIN_SECONDS and not near_end else 0
|
||||
|
||||
|
||||
def not_found(what: str = "generic") -> HTTPException:
|
||||
"""The 404 for a row that is gone — structured, so the bilingual UI says it in the reader's
|
||||
@@ -128,7 +164,7 @@ async def capped_stream_to(
|
||||
|
||||
|
||||
def issue_refused(exc: gitea.GiteaError) -> HTTPException:
|
||||
"""The refusal an issue-tracker failure (E16) answers as — the send, the preview and the
|
||||
"""The refusal an issue-tracker failure answers as — the send, the preview and the
|
||||
admin's connection test alike. Literal reasons, one per case: the frontend's contract test reads
|
||||
them to require a sentence for each in both languages."""
|
||||
match exc.reason:
|
||||
|
||||
+25
-134
@@ -1,5 +1,4 @@
|
||||
"""Admin-only onboarding: review access requests (Invites), approve/deny, manual add.
|
||||
Also the demo-account admin surface: the demo email whitelist + a manual state reset."""
|
||||
"""Admin-only onboarding: review access requests (Invites), approve/deny, manual add."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime, timezone
|
||||
@@ -19,15 +18,12 @@ from app.auth import (
|
||||
count_admins,
|
||||
env_allowed_canonicals,
|
||||
find_user_by_email,
|
||||
get_or_create_demo_user,
|
||||
is_allowed,
|
||||
operator_contact,
|
||||
purge_user,
|
||||
)
|
||||
from app.db import get_db
|
||||
from app.demo import reset_demo_state
|
||||
from app.models import (
|
||||
DemoWhitelist,
|
||||
Invite,
|
||||
User,
|
||||
)
|
||||
@@ -131,7 +127,7 @@ def _welcome_mail_parts(db: Session, email: str) -> tuple[str | None, bool, int]
|
||||
"""
|
||||
hours = int(APPROVAL_VERIFY_TTL.total_seconds() // 3600)
|
||||
user = find_user_by_email(db, email)
|
||||
if user is None or user.is_demo:
|
||||
if user is None:
|
||||
return None, False, hours
|
||||
if user.email_verified or not email_mod.email_enabled(db):
|
||||
return None, True, hours
|
||||
@@ -142,20 +138,21 @@ def _activate_user_by_email(db: Session, email: str) -> None:
|
||||
"""Approving access activates a matching (e.g. pending password-registration) account so
|
||||
it can sign in. No-op if no such user exists yet (a Google user is created at first login).
|
||||
|
||||
E9.5 S1: this path deliberately does NOT revoke a preset password. Revoking here destroyed the
|
||||
This path deliberately does NOT revoke a preset password. Revoking here destroyed the
|
||||
password a legitimate registrant had just chosen and made "your password was cleared for
|
||||
security" the DEFAULT experience, because approve-before-verify is the normal ordering.
|
||||
|
||||
S1 justified that by saying `password_login` refuses an unverified account BEFORE it looks at
|
||||
`is_active`, so a pre-emptor could not sign in "approved or not". A review of the finished epic
|
||||
showed that argument only holds while the address stays unverified — and S6's approval mail
|
||||
hands the real owner a confirmation link precisely so they will end that state. Nothing replaced
|
||||
the revoke, and that is deliberate: round 2 of the same review tried "the newest registration
|
||||
owns an unverified account" and reverted it, because the rule is symmetric — an attacker can
|
||||
always register last — and it handed anonymous callers a write onto a real account's credential.
|
||||
The owner's recourse is password-reset, which proves the mailbox, rewrites the hash and bumps
|
||||
the session epoch. The residual is logged as SEC-16. The revoke also stays on the Google-adoption path
|
||||
(auth.revoke_preset_password_if_unverified), where ownership is actually attested."""
|
||||
That was once justified by saying `password_login` refuses an unverified account BEFORE it
|
||||
looks at `is_active`, so a pre-emptor could not sign in "approved or not". A review of the
|
||||
finished epic showed that argument only holds while the address stays unverified — and the
|
||||
approval mail hands the real owner a confirmation link precisely so they will end that state.
|
||||
Nothing replaced the revoke, and that is deliberate: the same review later tried "the newest
|
||||
registration owns an unverified account" and reverted it, because the rule is symmetric — an
|
||||
attacker can always register last — and it handed anonymous callers a write onto a real
|
||||
account's credential. The owner's recourse is password-reset, which proves the mailbox, rewrites
|
||||
the hash and bumps the session epoch. The residual is logged as SEC-16. The revoke also stays on
|
||||
the Google-adoption path (auth.revoke_untrusted_preset_password), where ownership is
|
||||
actually attested."""
|
||||
# Canonical, with the literal spelling preferred when both exist: approving
|
||||
# `npeter83.temu@gmail.com` must activate the account that registered as `npeter83temu@gmail.com`,
|
||||
# since Gmail delivers both to one mailbox and the admin decided about the mailbox.
|
||||
@@ -400,19 +397,18 @@ def _serialize_user(u: User) -> dict:
|
||||
"is_active": u.is_active,
|
||||
"is_suspended": u.is_suspended,
|
||||
"email_verified": u.email_verified,
|
||||
"is_demo": u.is_demo,
|
||||
"has_password": u.password_hash is not None,
|
||||
# Distinguishes "this account never had a password" (Google-only) from "SEC-10 cleared it" —
|
||||
# the same distinction _send_reset_if_eligible needs, and the one an admin has to see before
|
||||
# deciding what kind of help someone needs.
|
||||
"password_revoked_at": iso(u.password_revoked_at),
|
||||
"has_google": u.google_sub is not None,
|
||||
# Only when it DIFFERS from the account address (E9.5 S3): "connected" is not the same
|
||||
# Only when it DIFFERS from the account address: "connected" is not the same
|
||||
# statement as "connected as someone else", and the second one is the one that explains a
|
||||
# confusing sign-in.
|
||||
"google_email": u.google_email if u.google_email != u.email else None,
|
||||
"created_at": iso(u.created_at),
|
||||
# NULL = no sign-in since the column existed (0085), or the shared demo account.
|
||||
# NULL = no sign-in since the column existed (0085).
|
||||
"last_login_at": iso(u.last_login_at),
|
||||
}
|
||||
|
||||
@@ -435,14 +431,12 @@ def set_user_role(
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Promote/demote a user. Guards: can't change your own role, can't touch the demo
|
||||
account, and can't remove the last remaining admin. Emails the user when it actually changes."""
|
||||
"""Promote/demote a user. Guards: can't change your own role and can't remove the
|
||||
last remaining admin. Emails the user when it actually changes."""
|
||||
role = body.role
|
||||
if role not in ("user", "admin"):
|
||||
raise refused(400, "invalid_role")
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise refused(400, "demo_role")
|
||||
if target.id == admin.id:
|
||||
raise refused(400, "own_role")
|
||||
if (
|
||||
@@ -490,14 +484,11 @@ def set_user_suspended(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Suspend or un-suspend an account. A suspended user can't sign in by any method and any
|
||||
live session is dropped on its next request. Guards: can't suspend the demo account, can't
|
||||
suspend yourself, and can't suspend the last active admin (that would lock admin out). On
|
||||
live session is dropped on its next request. Guards: can't suspend yourself, and can't suspend the last active admin (that would lock admin out). On
|
||||
un-suspend the user is emailed they can return (suspend is announced reactively, on a blocked
|
||||
sign-in)."""
|
||||
suspended = body.suspended
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise refused(400, "demo_suspend")
|
||||
if target.id == admin.id:
|
||||
raise refused(400, "own_suspend")
|
||||
if (
|
||||
@@ -538,10 +529,8 @@ def admin_delete_user(
|
||||
) -> dict:
|
||||
"""Admin-triggered account deletion — the same full erasure a user can perform on themselves
|
||||
(cascade delete of all personal data + access-request cleanup + Google-grant revoke). Guards:
|
||||
not the demo account, not yourself (use Settings → Account), and not the last admin."""
|
||||
not yourself (use Settings → Account), and not the last admin."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise refused(400, "demo_delete")
|
||||
if target.id == admin.id:
|
||||
raise refused(400, "own_delete")
|
||||
if (
|
||||
@@ -563,10 +552,7 @@ def admin_delete_user(
|
||||
return {"deleted": user_id}
|
||||
|
||||
|
||||
# --- Demo account: email whitelist + state reset -------------------------------------
|
||||
|
||||
|
||||
# --- Stuck sign-ups: the admin lifeline (E9.5 S5, roadmap UX-25) ---------------------------------
|
||||
# --- Stuck sign-ups: the admin lifeline -----------------------------------------------------------
|
||||
|
||||
|
||||
def _pending_access_rows(db: Session) -> list[User]:
|
||||
@@ -587,7 +573,6 @@ def _pending_access_rows(db: Session) -> list[User]:
|
||||
db.execute(
|
||||
select(User)
|
||||
.where(
|
||||
User.is_demo.is_(False),
|
||||
User.is_suspended.is_(False),
|
||||
or_(User.is_active.is_(False), User.email_verified.is_(False)),
|
||||
)
|
||||
@@ -625,7 +610,7 @@ def user_access(
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Everything the account panel shows about why this person can or cannot get in (E12 S5).
|
||||
"""Everything the account panel shows about why this person can or cannot get in.
|
||||
|
||||
Its own endpoint rather than extra columns on /users: `is_allowed` and `can_activate` each run a
|
||||
query, and the panel opens for ONE user at a time — folding them into the list would pay that cost
|
||||
@@ -644,7 +629,7 @@ def create_set_password_link(
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Mint a one-time link that lets this account set a password, with no email involved (E12 S5).
|
||||
"""Mint a one-time link that lets this account set a password, with no email involved.
|
||||
|
||||
The answer to SEC-15, SEC-16 and "no SMTP and someone forgot their password" at once. The admin
|
||||
hands the link over by whatever channel they trust; opening it sets a password and signs out
|
||||
@@ -653,13 +638,8 @@ def create_set_password_link(
|
||||
Deliberately NOT gated on the account's state. It is useful for a pre-empted address, for a
|
||||
revoked password, for a Google-only account that wants one, and for a plain forgotten password on
|
||||
an instance with no mail — narrowing it to one of those would only make the admin's real case the
|
||||
one it refuses. The demo account is the single exception: it is shared, so handing anyone a way to
|
||||
change its password is handing them everyone else's session."""
|
||||
one it refuses."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise HTTPException(
|
||||
status_code=400, detail={"code": "setpw", "reason": "demo", "email": target.email}
|
||||
)
|
||||
url, expires_at = issue_set_password_link(db, target)
|
||||
# The URL itself is NOT recorded: the audit log is read by humans and kept long, and a live
|
||||
# credential-setting link is the last thing that belongs in it. That the link was issued, by whom,
|
||||
@@ -692,8 +672,6 @@ def activate_user(
|
||||
already do exactly this by re-entering the address in the whitelist box (`add_invite` →
|
||||
`_activate_user_by_email`). What was missing was any way to know they had to."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo:
|
||||
raise refused(400, "demo_activate")
|
||||
if target.is_active:
|
||||
return _serialize_user(target) # idempotent: a double click is not an error
|
||||
if not is_allowed(db, target.email):
|
||||
@@ -731,7 +709,7 @@ def resend_user_verification(
|
||||
Not rate-limited: an admin is not a threat model to enumerate against, and the deliberate silence
|
||||
of the public endpoint is exactly what this exists to escape."""
|
||||
target = get_or_404(db, User, user_id, what="user")
|
||||
if target.is_demo or target.email_verified:
|
||||
if target.email_verified:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail={"code": "resend", "reason": "not_applicable", "email": target.email},
|
||||
@@ -757,93 +735,6 @@ def resend_user_verification(
|
||||
return {"ok": True, "email": target.email}
|
||||
|
||||
|
||||
def _serialize_demo(row: DemoWhitelist) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
"email": row.email,
|
||||
"note": row.note,
|
||||
"added_by": row.added_by,
|
||||
"created_at": iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/demo/whitelist")
|
||||
def list_demo_whitelist(_: User = Depends(admin_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
rows = (
|
||||
db.execute(select(DemoWhitelist).order_by(DemoWhitelist.created_at.desc())).scalars().all()
|
||||
)
|
||||
return [_serialize_demo(r) for r in rows]
|
||||
|
||||
|
||||
class AddDemoWhitelistIn(BaseModel):
|
||||
email: str = ""
|
||||
note: str | None = None
|
||||
|
||||
|
||||
@router.post("/demo/whitelist")
|
||||
def add_demo_whitelist(
|
||||
body: AddDemoWhitelistIn,
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Add an email that may enter the shared demo account from the login page (no Google
|
||||
sign-in). Idempotent — re-adding an existing email just returns it."""
|
||||
email = body.email.strip().lower()
|
||||
if not valid_email(email):
|
||||
raise refused(400, "invalid_email")
|
||||
row = db.execute(select(DemoWhitelist).where(DemoWhitelist.email == email)).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = DemoWhitelist(
|
||||
email=email,
|
||||
note=(body.note or "").strip() or None,
|
||||
added_by=admin.email,
|
||||
)
|
||||
db.add(row)
|
||||
audit.record(
|
||||
db, admin.id, AuditAction.DEMO_ADD, target_type="demo", target_id=email, summary=email
|
||||
)
|
||||
db.commit()
|
||||
return _serialize_demo(row)
|
||||
|
||||
|
||||
@router.delete("/demo/whitelist/{entry_id}")
|
||||
def remove_demo_whitelist(
|
||||
entry_id: int,
|
||||
admin: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
row = db.get(DemoWhitelist, entry_id)
|
||||
if row is not None:
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.DEMO_REMOVE,
|
||||
target_type="demo",
|
||||
target_id=row.email,
|
||||
summary=row.email,
|
||||
)
|
||||
db.delete(row)
|
||||
db.commit()
|
||||
return {"deleted": entry_id}
|
||||
|
||||
|
||||
@router.post("/demo/reset")
|
||||
def reset_demo(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Manually clean up the shared demo sandbox back to a baseline. A scheduled job does the
|
||||
same automatically (admin-tunable interval); this is the admin's on-demand button."""
|
||||
demo = get_or_create_demo_user(db)
|
||||
seeded = reset_demo_state(db, demo)
|
||||
audit.record(
|
||||
db,
|
||||
admin.id,
|
||||
AuditAction.DEMO_RESET,
|
||||
summary=f"{seeded} sample playlists re-seeded",
|
||||
after={"playlists_seeded": seeded}, # the count as data: named in the reader's language
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "reset": True, "playlists_seeded": seeded}
|
||||
|
||||
|
||||
@router.post("/purge-discovery")
|
||||
def purge_discovery(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Reclaim all un-kept discovery content NOW (ignoring the grace period): live-search result
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The admin diagnostics console's read side (E22.6): query `log_events`, the server's stored log.
|
||||
"""The admin diagnostics console's read side: query `log_events`, the server's stored log.
|
||||
|
||||
Admin-only and read-only — rows leave the table through the retention job (`log_retention_days`),
|
||||
never through here. Reading is not audited: it changes nothing, and the console's live tail polls,
|
||||
|
||||
+90
-139
@@ -1,6 +1,6 @@
|
||||
"""Auth HTTP endpoints (OAuth login/callback, register/verify, password, session mgmt).
|
||||
|
||||
Split out of app/auth.py (R12 S2); the core auth surface (current_user, scopes, token
|
||||
Split out of app/auth.py; the core auth surface (current_user, scopes, token
|
||||
helpers) stays in app/auth.py, imported below."""
|
||||
|
||||
import secrets
|
||||
@@ -29,8 +29,8 @@ from app.auth import (
|
||||
WRITE_SCOPES,
|
||||
current_user,
|
||||
env_admin_canonicals,
|
||||
env_allowed_canonicals,
|
||||
find_user_by_email,
|
||||
get_or_create_demo_user,
|
||||
google_enabled,
|
||||
is_allowed,
|
||||
log,
|
||||
@@ -39,7 +39,7 @@ from app.auth import (
|
||||
)
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import AuthToken, DemoWhitelist, Invite, OAuthToken, User, display_text
|
||||
from app.models import AuthToken, Invite, OAuthToken, User, display_text
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import encrypt, hash_password, hash_token, verify_password
|
||||
from app.utils import canonical_email, valid_email
|
||||
@@ -53,7 +53,7 @@ VERIFY_TTL = timedelta(hours=24)
|
||||
# in and sets no password.
|
||||
APPROVAL_VERIFY_TTL = timedelta(days=7)
|
||||
RESET_TTL = timedelta(hours=1)
|
||||
# An admin-issued set-password link (E12 S5) travels out of band — the admin has to actually reach the
|
||||
# An admin-issued set-password link travels out of band — the admin has to actually reach the
|
||||
# person and hand it over — so the one-hour reset window is far too short. A day matches VERIFY_TTL and
|
||||
# the affordance the admin was shown.
|
||||
ADMIN_SETPW_TTL = timedelta(hours=24)
|
||||
@@ -72,7 +72,6 @@ _reset_limiter = RateLimiter(max_events=5, window_seconds=300)
|
||||
# Separate from _reset_limiter on purpose: opening a link must not eat the budget for REQUESTING one.
|
||||
_reset_check_limiter = RateLimiter(max_events=10, window_seconds=300)
|
||||
_verify_resend_limiter = RateLimiter(max_events=5, window_seconds=300)
|
||||
_request_access_limiter = RateLimiter(max_events=5, window_seconds=300)
|
||||
_suspend_email_limiter = RateLimiter(max_events=1, window_seconds=3600)
|
||||
|
||||
|
||||
@@ -92,8 +91,6 @@ def _notify_suspended(background: BackgroundTasks, user: User) -> None:
|
||||
)
|
||||
|
||||
|
||||
_demo_limiter = RateLimiter(max_events=5, window_seconds=60)
|
||||
|
||||
router = APIRouter(prefix="/auth", tags=["auth"])
|
||||
|
||||
_oauth = OAuth()
|
||||
@@ -161,24 +158,12 @@ def notify_admins_of_request(
|
||||
background.add_task(email_mod.send_admin_new_request, recipients, lang, requester)
|
||||
|
||||
|
||||
def is_demo_allowed(db: Session, email: str) -> bool:
|
||||
"""Whether this email may enter the shared demo account (no Google sign-in)."""
|
||||
if not email:
|
||||
return False
|
||||
return (
|
||||
db.execute(
|
||||
select(DemoWhitelist).where(DemoWhitelist.email == email.lower())
|
||||
).scalar_one_or_none()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
class InviteRequest(NamedTuple):
|
||||
"""What an access request settled on.
|
||||
|
||||
`invite` is the row the request now lives under — which is NOT always the address that was just
|
||||
typed: one Gmail mailbox spelled two ways settles on whichever row already exists, and that row's
|
||||
address is where mail will go. Callers that tell the user what happened need it (E9.5 S4).
|
||||
address is where mail will go. Callers that tell the user what happened need it.
|
||||
`is_new` is the old boolean return in a name: true only when a row was created or a denied one
|
||||
reopened, i.e. exactly when admins should hear about it. A repeat request must not re-spam them.
|
||||
"""
|
||||
@@ -196,14 +181,14 @@ def upsert_pending_invite(db: Session, email: str, lang: str | None = None) -> I
|
||||
not reopen a decision that has already been made for that mailbox. The row is still STORED under
|
||||
the address as typed, because that is where mail has to go.
|
||||
|
||||
`lang` is the language of the surface the request came from — the sign-in page, or the locale
|
||||
Google reported for a refused sign-in. The latest one wins: someone who asks again from a
|
||||
Hungarian page is telling us something newer than last time. Anything unsupported is ignored
|
||||
`lang` is the language the request came in — the locale Google reported for a refused sign-in.
|
||||
The latest one wins: someone who asks again under another locale is telling us something newer
|
||||
than last time. Anything unsupported is ignored
|
||||
rather than stored, so a forged body cannot put junk in the column.
|
||||
|
||||
The commit contract is unchanged: this still commits ONLY when it writes something. Callers rely
|
||||
on that — `_register_account` persists its own flushed user precisely because a no-op here
|
||||
commits nothing (E9.5 S6).
|
||||
commits nothing.
|
||||
"""
|
||||
email = (email or "").lower()
|
||||
if not valid_email(email):
|
||||
@@ -215,6 +200,12 @@ def upsert_pending_invite(db: Session, email: str, lang: str | None = None) -> I
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
# An address the env already lets in has nothing to ask for, and a pending row for it would be
|
||||
# worse than nothing: `is_allowed` answers from the invite rows once any exist, so a stranger
|
||||
# registering a sibling spelling (`b.ob@` beside an env-granted `bob@`) would lock the real
|
||||
# owner out of a grant no admin could see.
|
||||
if not rows and canon in env_allowed_canonicals():
|
||||
return InviteRequest(None, False)
|
||||
# An approved row wins, so a denied duplicate can't mask an approval. Then a row that is not
|
||||
# DENIED, preferring the exact spelling — a denied sibling must never be picked while a pending
|
||||
# row exists, or the reopen branch below would fire and put one person in the admin's queue
|
||||
@@ -254,7 +245,7 @@ def _access_notice_url(kind: str, email: str = "", inv: Invite | None = None) ->
|
||||
|
||||
It rides the FRAGMENT, not the query string, for the same reason the verify/reset tokens do
|
||||
(SB3): a fragment never reaches the server, a proxy access log or a Referer header — and this
|
||||
notice now carries the user's address. Naming it is the point of E9.5 S4: on 2026-09-03 a
|
||||
notice now carries the user's address. Naming it is the point: on 2026-09-03 a
|
||||
registration as `npeter83.temu@` met an OAuth claim of `npeter83temu@`, and a screen that said
|
||||
"request recorded" without saying FOR WHAT hid the mismatch for a day.
|
||||
|
||||
@@ -306,9 +297,10 @@ class EmailPasswordIn(BaseModel):
|
||||
password: str | None = None
|
||||
# The language the sign-up page was in. Captured at registration because it is the ONLY moment
|
||||
# the choice exists anywhere the server can see: it lives in the browser until an account does,
|
||||
# and a preference nobody recorded cannot be backfilled later (the lesson `google_email` taught
|
||||
# in E9.5 S3 — those rows still read NULL until each user signs in again). Nothing reads it yet;
|
||||
# the bilingual mail templates are E9.5 S7.
|
||||
# and a preference nobody recorded cannot be backfilled later (as `google_email` showed: those
|
||||
# rows still read NULL until each user signs in again). It is stored as the account's
|
||||
# `preferences["language"]`, which the UI starts in and every mail to the account is written in
|
||||
# (`mailtext.lang_for_user` / `lang_for_email`).
|
||||
lang: str | None = None
|
||||
|
||||
|
||||
@@ -408,7 +400,7 @@ async def callback(
|
||||
return RedirectResponse(url=_access_notice_url("requested", email, req.invite))
|
||||
return RedirectResponse(url=_access_notice_url("denied"))
|
||||
|
||||
user = db.query(User).filter(User.google_sub == sub).one_or_none()
|
||||
user = db.execute(select(User).where(User.google_sub == sub)).scalar_one_or_none()
|
||||
if user is None:
|
||||
# No account carries this Google identity yet. If one already exists for this MAILBOX —
|
||||
# e.g. an email+password registration — adopt the Google identity onto it instead of
|
||||
@@ -421,7 +413,11 @@ async def callback(
|
||||
# (ids 13 and 14). Exact spelling still wins when both exist, so an account that already
|
||||
# matches literally is never passed over for a canonical sibling.
|
||||
canon = canonical_email(email)
|
||||
candidates = db.query(User).filter(User.email_canonical == canon).order_by(User.id).all()
|
||||
candidates = (
|
||||
db.execute(select(User).where(User.email_canonical == canon).order_by(User.id))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
existing = next((u for u in candidates if u.email == email), None) or next(
|
||||
iter(candidates), None
|
||||
)
|
||||
@@ -447,9 +443,10 @@ async def callback(
|
||||
return RedirectResponse(url=_access_notice_url("denied"))
|
||||
user = existing
|
||||
user.google_sub = sub
|
||||
# SEC-10: drop any password pre-set on this account while its email was still unverified,
|
||||
# now that Google attests ownership (bumps the epoch to revoke any session it minted).
|
||||
revoke_preset_password_if_unverified(user)
|
||||
# SEC-10: drop any password nobody proved the mailbox for, now that Google attests
|
||||
# ownership (bumps the epoch to revoke any session it minted). Runs before the
|
||||
# activation below, which is what it reads.
|
||||
revoke_untrusted_preset_password(user)
|
||||
# Google verified the email and is_allowed granted access, so finish activating a
|
||||
# previously pending password registration (otherwise current_user would bounce it).
|
||||
user.email_verified = True
|
||||
@@ -480,7 +477,16 @@ async def callback(
|
||||
# default for a brand-new account (its role attribute is still unset before flush).
|
||||
# Canonical, like every other read of the env lists: a dotted or `+tag`ged ADMIN_EMAILS entry
|
||||
# would otherwise never bootstrap its admin, because Google answers with the plain primary.
|
||||
#
|
||||
# Becoming admin this way drops any password the account holds, on whichever sign-in it
|
||||
# happens. Nothing here can tell who chose that password: without SMTP registration marks the
|
||||
# address verified with no proof (and SMTP configured later never revisits it), and an admin's
|
||||
# Activate vouches for access, not for the password. So a stranger who registered the address
|
||||
# first, and was let in, would otherwise see their own live session become an admin one. The
|
||||
# real owner, signed in through Google right now, sets a new password in Settings.
|
||||
if canonical_email(email) in env_admin_canonicals():
|
||||
if user.role != "admin" and user.password_hash is not None:
|
||||
_revoke_password(user)
|
||||
user.role = "admin"
|
||||
elif not user.role:
|
||||
user.role = "user"
|
||||
@@ -515,7 +521,7 @@ def _sync_google_email(db: Session, user: User, email: str) -> None:
|
||||
of the spelling they chose, because sign-in matches the stored address first. Google returns the
|
||||
account's PRIMARY address, which need not be what they registered with — and can never contain a
|
||||
`+` at all — so without this guard every plus-tagged registration lost its tag on the first Google
|
||||
sign-in. (E9.5 S3 made that reachable by matching adoption across spellings; the rewrite itself is
|
||||
sign-in. (Matching adoption across spellings made that reachable; the rewrite itself is
|
||||
older.)
|
||||
|
||||
Never overwrite with an address another account already owns either: that would hit the unique
|
||||
@@ -525,7 +531,9 @@ def _sync_google_email(db: Session, user: User, email: str) -> None:
|
||||
return
|
||||
if canonical_email(user.email) == canonical_email(email):
|
||||
return # same mailbox, their spelling — leave it alone
|
||||
clash = db.query(User).filter(User.email == email, User.id != user.id).one_or_none()
|
||||
clash = db.execute(
|
||||
select(User).where(User.email == email, User.id != user.id)
|
||||
).scalar_one_or_none()
|
||||
if clash is None:
|
||||
user.email = email
|
||||
|
||||
@@ -582,9 +590,11 @@ def _complete_link(
|
||||
if link_uid != request.session.get("user_id") and link_uid not in accounts:
|
||||
return RedirectResponse(url="/?link=error")
|
||||
user = db.get(User, link_uid)
|
||||
if user is None or user.is_demo:
|
||||
if user is None:
|
||||
return RedirectResponse(url="/?link=error")
|
||||
other = db.query(User).filter(User.google_sub == sub, User.id != user.id).one_or_none()
|
||||
other = db.execute(
|
||||
select(User).where(User.google_sub == sub, User.id != user.id)
|
||||
).scalar_one_or_none()
|
||||
if other is not None:
|
||||
return RedirectResponse(url="/?link=conflict")
|
||||
if user.google_sub is not None and user.google_sub != sub:
|
||||
@@ -628,73 +638,12 @@ async def logout(request: Request):
|
||||
return JSONResponse({"ok": True, "switched": False})
|
||||
|
||||
|
||||
class AccessRequestIn(EmailIn):
|
||||
# The language of the page this request was made from. Same reasoning as registration's `lang`
|
||||
# (and the same shape): it is the ONLY moment this preference exists anywhere the server can
|
||||
# see, since the requester has no account to carry it, and it decides what language the approval
|
||||
# or denial mail is written in.
|
||||
lang: str | None = None
|
||||
|
||||
|
||||
@router.post("/request-access")
|
||||
async def request_access(
|
||||
body: AccessRequestIn,
|
||||
request: Request,
|
||||
background: BackgroundTasks,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Public: ask for access. Idempotent — repeat requests for the same email don't
|
||||
duplicate or re-spam admins (upsert answers `is_new=False` for an already-pending row).
|
||||
|
||||
Answers a uniform ``"pending"`` for EVERY outcome — already-allowed, newly-pending,
|
||||
already-pending, and rate-limited alike. The old ``"approved"`` reply for an allow-listed
|
||||
address made this a public enumeration oracle for the allow-list (which the rest of the module
|
||||
is careful to avoid, cf. demo_login); an already-allowed user can still just sign in with Google.
|
||||
Rate-limited per client IP like register/login/reset, and a block returns "pending" too so the
|
||||
endpoint can't be probed for throttling either."""
|
||||
email = (body.email or "").strip().lower()
|
||||
if not valid_email(email):
|
||||
raise refused(400, "invalid_email")
|
||||
if not _request_access_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "status": "pending"}
|
||||
if is_allowed(db, email):
|
||||
return {"ok": True, "status": "pending"} # already allowed — don't reveal it
|
||||
req = upsert_pending_invite(db, email, body.lang)
|
||||
if req.is_new:
|
||||
notify_admins_of_request(db, background, email)
|
||||
return {"ok": True, "status": "pending"}
|
||||
|
||||
|
||||
@router.post("/demo")
|
||||
def demo_login(body: EmailIn, request: Request, db: Session = Depends(get_db)) -> dict:
|
||||
"""Hidden demo entry: a whitelisted email logs straight into the shared demo account,
|
||||
no Google OAuth. The login page fires this quietly (debounced) as the user types/pastes,
|
||||
so there is no visible 'demo login' button. Rate-limited per IP; when blocked it answers
|
||||
exactly like a non-match (authenticated=false) so the field can't be hammered as an
|
||||
enumeration oracle. On a match it sets the session and the client reloads into the app."""
|
||||
if not _demo_limiter.allow(client_key(request)):
|
||||
return {"ok": True, "authenticated": False}
|
||||
email = (body.email or "").strip().lower()
|
||||
if valid_email(email) and is_demo_allowed(db, email):
|
||||
demo = get_or_create_demo_user(db)
|
||||
# Isolate the demo: drop any real-account wallet on this tab first, so the demo session
|
||||
# can't switch to / act as a previously signed-in account via the multi-account header.
|
||||
request.session.clear()
|
||||
request.session["user_id"] = demo.id
|
||||
_remember_epoch(request, demo)
|
||||
# The demo key IS an address, so it is not in the line — see the denied branch above.
|
||||
log.info("signed in", extra=logs.event("auth.login.ok", method="demo", uid=demo.id))
|
||||
return {"ok": True, "authenticated": True}
|
||||
return {"ok": True, "authenticated": False}
|
||||
|
||||
|
||||
def _establish_session(request: Request, user: User) -> None:
|
||||
"""Sign `user` in for this browser session — the ONE sign-in path, shared by the Google callback
|
||||
and password login, so both join the same account switcher and both stamp `last_login_at` (the
|
||||
caller commits). Multi-session: every account that authenticated in this browser is remembered so
|
||||
the user can switch between them; only accounts that actually proved themselves HERE land in that
|
||||
wallet (it is the proof they may be switched to). The demo entry deliberately does not come
|
||||
through here: it clears the wallet instead, and its shared account has no meaningful last login."""
|
||||
wallet (it is the proof they may be switched to)."""
|
||||
accounts = [a for a in (request.session.get("account_ids") or []) if a != user.id]
|
||||
accounts.append(user.id)
|
||||
request.session["account_ids"] = accounts[-MAX_SESSION_ACCOUNTS:]
|
||||
@@ -784,7 +733,7 @@ def _consume_password_token(db: Session, raw: str | None) -> tuple[User, str] |
|
||||
|
||||
|
||||
def issue_set_password_link(db: Session, user: User) -> tuple[str, datetime]:
|
||||
"""An admin-issued, single-use link that lets `user` set a password without any email (E12 S5).
|
||||
"""An admin-issued, single-use link that lets `user` set a password without any email.
|
||||
|
||||
One mechanism answers three things: SEC-15 (on a no-SMTP instance every registration is
|
||||
auto-verified, so a pre-empted address cannot be recovered by mail), SEC-16 (a pre-empted address
|
||||
@@ -866,10 +815,10 @@ def _register_account(email: str, password: str, lang: str | None = None) -> Non
|
||||
# for the same input — so it is not a new capability, just the end of a dead end. An
|
||||
# account that is already confirmed still gets nothing: there is nothing to confirm.
|
||||
#
|
||||
# And a re-send is ALL it does: the submitted password is discarded. E9.5's closing
|
||||
# And a re-send is ALL it does: the submitted password is discarded. A closing
|
||||
# review tried "while the address is unverified, the newest registration owns it"
|
||||
# (replace the hash, kill the outstanding verify tokens) as a fix for address
|
||||
# pre-emption, and round 2 showed it was worse than the problem — DO NOT REPEAT IT.
|
||||
# pre-emption, and it proved worse than the problem — DO NOT REPEAT IT.
|
||||
# The rule is symmetric, and an attacker can always be last: it hands an unauthenticated
|
||||
# caller the power to overwrite a real registrant's chosen password and delete the
|
||||
# confirmation link already in their inbox, and on a SEC-10-stripped row (password_hash
|
||||
@@ -878,7 +827,7 @@ def _register_account(email: str, password: str, lang: str | None = None) -> Non
|
||||
# primitive: password-reset accepts this account, proves the mailbox, rewrites the hash
|
||||
# and bumps the session epoch. The residual is logged as SEC-16.
|
||||
user = existing
|
||||
if not user.email_verified and not user.is_demo and email_mod.email_enabled():
|
||||
if not user.email_verified and email_mod.email_enabled():
|
||||
send_verification_link(db, user)
|
||||
return
|
||||
# Without working SMTP we can't deliver a verification link, so email ownership can't gate
|
||||
@@ -918,7 +867,7 @@ def _register_account(email: str, password: str, lang: str | None = None) -> Non
|
||||
db.commit()
|
||||
if email_ok:
|
||||
send_verification_link(db, user)
|
||||
# Fired unconditionally, unlike the other two callers which gate on a fresh invite row: for
|
||||
# Fired unconditionally, unlike the Google callback, which gates on a fresh invite row: for
|
||||
# an address the admin already approved, nothing lands in the Access-requests queue, so this
|
||||
# is the only place the account's existence is announced at all. Delivery is a separate
|
||||
# matter — with no working SMTP `_send` drops it, which is why the pre-approved registrant
|
||||
@@ -928,7 +877,7 @@ def _register_account(email: str, password: str, lang: str | None = None) -> Non
|
||||
|
||||
def issue_verification_link(db: Session, user: User, ttl: timedelta | None = None) -> str:
|
||||
"""A fresh confirmation link for `user`, returned rather than sent — for a mail that carries the
|
||||
link inside a larger message (the branching approval/invitation mails, E9.5 S6).
|
||||
link inside a larger message (the branching approval/invitation mails).
|
||||
|
||||
Same token machinery and the same fragment (SB3) as the standalone confirmation mail, so the two
|
||||
kinds of link cannot drift apart in shape or in how they are consumed."""
|
||||
@@ -943,8 +892,8 @@ def send_verification_link(db: Session, user: User) -> bool:
|
||||
address is the one its owner chose to be reachable at, and a `+tag` there is usually deliberate.
|
||||
The token rides the URL FRAGMENT (#) so it can never reach a proxy/access log or a Referer (SB3).
|
||||
|
||||
One definition, three callers — registration, the self-service resend, and the admin-side resend
|
||||
(E9.5 S5). The link's shape and TTL must not drift between them; a verification mail that differs
|
||||
One definition, three callers — registration, the self-service resend, and the admin-side resend.
|
||||
The link's shape and TTL must not drift between them; a verification mail that differs
|
||||
by who triggered it is a bug waiting to be discovered by exactly one user.
|
||||
|
||||
Returns whether the mail actually went out, so a caller with a human waiting on the answer (the
|
||||
@@ -967,8 +916,8 @@ def verify_email_confirm(body: TokenIn, db: Session = Depends(get_db)) -> dict:
|
||||
dropped the password this account registered with, so sign-in would answer with the deliberately
|
||||
uniform "Invalid email or password" — true, but unreadable to the person who just proved they own
|
||||
the mailbox. Saying so leaks nothing: holding a valid single-use verify token for this account
|
||||
already proves that much. Since E9.5 S1 the common source of that state is gone (admin approval no
|
||||
longer revokes), leaving accounts stripped before that change and the Google-adoption path, so this
|
||||
already proves that much. The common source of that state is gone (admin approval no longer
|
||||
revokes), leaving accounts stripped before that change and the Google-adoption path, so this
|
||||
flag is rare now — but it must stay, because those users have no other way back in."""
|
||||
user = _consume_token(db, body.token, "verify")
|
||||
if user is None:
|
||||
@@ -1012,7 +961,7 @@ def _resend_verify_if_eligible(email: str) -> None:
|
||||
return
|
||||
with SessionLocal() as db:
|
||||
user = find_user_by_email(db, email)
|
||||
if user is not None and not user.email_verified and not user.is_demo:
|
||||
if user is not None and not user.email_verified:
|
||||
send_verification_link(db, user)
|
||||
|
||||
|
||||
@@ -1028,7 +977,7 @@ def _login_refused(status: int, reason: str, user: User | None = None, **extra)
|
||||
that proof (`invalid`, `rate_limited`) carry nothing at all, so they stay byte-identical for a
|
||||
real account and an unknown one.
|
||||
|
||||
Since E9.5 S1 the two gates are order-independent, so `approved` really can be true while
|
||||
The two gates are order-independent, so `approved` really can be true while
|
||||
`email_verified` is false, and vice versa — the SPA renders whichever is actually outstanding
|
||||
rather than assuming a sequence.
|
||||
"""
|
||||
@@ -1058,11 +1007,11 @@ def password_login(
|
||||
email = (body.email or "").strip().lower()
|
||||
password = body.password or ""
|
||||
user = find_user_by_email(db, email)
|
||||
# Always run the KDF (against a decoy hash for an unknown / passwordless / demo account) so the
|
||||
# Always run the KDF (against a decoy hash for an unknown / passwordless account) so the
|
||||
# response time is the same whether or not the email exists — no timing-based enumeration.
|
||||
candidate_hash = user.password_hash if (user and not user.is_demo) else None
|
||||
candidate_hash = user.password_hash if user else None
|
||||
password_ok = verify_password(password, candidate_hash or _DECOY_PASSWORD_HASH)
|
||||
if user is None or user.is_demo or not password_ok:
|
||||
if user is None or not password_ok:
|
||||
raise _login_refused(401, "invalid")
|
||||
if user.is_suspended:
|
||||
_notify_suspended(background, user)
|
||||
@@ -1101,7 +1050,7 @@ def password_reset_request(
|
||||
|
||||
def _send_reset_if_eligible(email: str) -> None:
|
||||
"""Background worker for password_reset_request: issue a reset token + email the link, but ONLY
|
||||
for a real (non-demo) password account. Runs after the response with its own DB session, so the
|
||||
for a password account. Runs after the response with its own DB session, so the
|
||||
request's timing never reveals whether the account exists. Token rides the URL FRAGMENT (#) so it
|
||||
can't leak into proxy/access logs or a Referer (SB3).
|
||||
|
||||
@@ -1112,7 +1061,7 @@ def _send_reset_if_eligible(email: str) -> None:
|
||||
proves identity through the session rather than by mail."""
|
||||
with SessionLocal() as db:
|
||||
user = find_user_by_email(db, email)
|
||||
if user is None or user.is_demo:
|
||||
if user is None:
|
||||
return
|
||||
if user.password_hash or user.password_revoked_at:
|
||||
raw = _issue_token(db, user, "reset", RESET_TTL)
|
||||
@@ -1200,37 +1149,49 @@ def bump_session_epoch(user: User) -> None:
|
||||
user.session_epoch = (user.session_epoch or 0) + 1
|
||||
|
||||
|
||||
def revoke_preset_password_if_unverified(user: User) -> None:
|
||||
def revoke_untrusted_preset_password(user: User) -> None:
|
||||
"""SEC-10 (account pre-emption): a password set on an account whose email was NEVER verified is
|
||||
untrusted — whoever registered the address (to pre-empt the real owner) could have chosen it. When
|
||||
the real owner attests ownership through Google, drop that password and bump the epoch to revoke any
|
||||
session it minted; the owner re-sets one via password-reset. No-op for an already-verified account,
|
||||
so a legit no-SMTP registration is unaffected.
|
||||
session it minted; the owner re-sets one via password-reset, or from Settings while the Google
|
||||
session lasts.
|
||||
|
||||
The Google-adoption callback is the ONLY caller (E9.5 S1). Admin approval used to call this too, and
|
||||
dropping it was right for the reason S1 gave — the revoke destroyed the password a legitimate
|
||||
A NEVER-ACTIVATED account is untrusted too, verified or not: without SMTP, registration marks the
|
||||
address verified with no proof at all. For an ADMIN_EMAILS / ALLOWED_EMAILS address no invite row
|
||||
gates the registration, so a stranger who registered it first would otherwise be activated here
|
||||
with their own password still working. (An account the callback makes ADMIN loses its password
|
||||
whatever its state — see the ADMIN_EMAILS block there.)
|
||||
|
||||
The Google-adoption callback is the ONLY caller. Admin approval used to call this too, and
|
||||
dropping it was right for the reason given — the revoke destroyed the password a legitimate
|
||||
registrant had just chosen, and since the admin normally decides before the user reads their mail,
|
||||
that made the "your password was cleared" detour the DEFAULT path rather than an edge case.
|
||||
|
||||
S1 also claimed the revoke was REDUNDANT there, because `password_login` refuses an unverified
|
||||
The revoke was also called REDUNDANT there, because `password_login` refuses an unverified
|
||||
account before it reads `is_active`. That part was wrong, and the epic's own review caught it: the
|
||||
gate holds only while the address is unverified, and S6's approval mail exists to get the real
|
||||
gate holds only while the address is unverified, and the approval mail exists to get the real
|
||||
owner to end that state — at which point a pre-emptor's password works. Nothing replaced the
|
||||
revoke, and that is deliberate. The review tried "the newest registration owns an unverified
|
||||
account" and reverted it: the rule is symmetric, so an attacker can simply register last, and it
|
||||
would have handed anonymous callers a write onto a real account's credential. What the owner
|
||||
actually has is password-reset, which proves the mailbox, rewrites the hash and bumps the epoch.
|
||||
The residual is logged as SEC-16; re-adding a caller here would put "your password was cleared"
|
||||
back in front of every legitimate registrant, which is the whole reason S1 removed it.
|
||||
back in front of every legitimate registrant, which is the whole reason it was removed.
|
||||
|
||||
`password_revoked_at` is what makes "the owner re-sets one via password-reset" actually true. The
|
||||
revoke leaves `password_hash IS NULL`, which is also what a Google-only account looks like — and
|
||||
_send_reset_if_eligible keyed off exactly that, so a revoked account silently got no reset link and
|
||||
was locked out for good (it could not sign in either). The stamp separates the two cases."""
|
||||
if not user.email_verified:
|
||||
user.password_hash = None
|
||||
user.password_revoked_at = datetime.now(timezone.utc)
|
||||
bump_session_epoch(user)
|
||||
if not user.email_verified or (not user.is_active and user.password_hash is not None):
|
||||
_revoke_password(user)
|
||||
|
||||
|
||||
def _revoke_password(user: User) -> None:
|
||||
"""Drop the account's password, stamp why (so password-reset still accepts it) and end every
|
||||
session it signed in."""
|
||||
user.password_hash = None
|
||||
user.password_revoked_at = datetime.now(timezone.utc)
|
||||
bump_session_epoch(user)
|
||||
|
||||
|
||||
@router.post("/logout-others")
|
||||
@@ -1239,9 +1200,7 @@ def logout_others(
|
||||
) -> dict:
|
||||
"""SA4: sign every OTHER session for this account out — bump the epoch so all existing cookies
|
||||
are rejected, then re-stamp THIS cookie so the caller stays signed in. For "I left myself logged
|
||||
in somewhere" / a suspected session compromise. The demo account (shared) is excluded."""
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
in somewhere" / a suspected session compromise."""
|
||||
# Commit the bump BEFORE re-stamping this cookie, so a failed commit can't strand the current
|
||||
# session at a newer epoch than the DB (see set_password).
|
||||
bump_session_epoch(user)
|
||||
@@ -1260,8 +1219,6 @@ async def link_google(
|
||||
"""Start linking a Google account to the signed-in (e.g. password) account. Requests only
|
||||
the identity scopes — YouTube access is granted separately via /auth/upgrade. The callback
|
||||
sees `oauth_link_uid` and attaches the identity instead of creating a new account."""
|
||||
if user.is_demo:
|
||||
return RedirectResponse(url="/")
|
||||
client = google_oauth(db)
|
||||
if client is None:
|
||||
return RedirectResponse(url="/?link=error")
|
||||
@@ -1287,8 +1244,6 @@ def set_password(
|
||||
"""Set or change the signed-in account's password. Setting a first password (Google-only
|
||||
account) needs no current password — the session already proves identity. Changing an
|
||||
existing one requires the current password, so a hijacked session can't silently rotate it."""
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo"})
|
||||
new_password = body.password or ""
|
||||
if len(new_password) < PASSWORD_MIN_LEN:
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
@@ -1319,10 +1274,6 @@ async def upgrade(
|
||||
(unsubscribe). The shared callback stores whatever Google returns, so `can_read` /
|
||||
`can_write` flip on once granted. Anything other than an explicit `write` defaults to
|
||||
the least-privileged read scope."""
|
||||
# This is the browser-facing redirect entry point, so the shared demo account is bounced
|
||||
# straight home (no YouTube link is ever attached to it) rather than shown a raw 403.
|
||||
if user.is_demo:
|
||||
return RedirectResponse(url="/")
|
||||
client = google_oauth(db)
|
||||
if client is None:
|
||||
return RedirectResponse(url="/")
|
||||
|
||||
@@ -12,7 +12,7 @@ from sqlalchemy.orm import Session
|
||||
from app.collation import collated, sorted_words
|
||||
from app.mailtext import lang_for_user
|
||||
from app import logs, quota
|
||||
from app.auth import admin_user, current_user, require_human, require_write_scope
|
||||
from app.auth import admin_user, current_user, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
@@ -67,8 +67,8 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_
|
||||
|
||||
# Per-channel aggregates over the shared catalog (channel_aggregates), computed on read rather than
|
||||
# denormalised: an Index Only Scan over a ~20 MB index, where it used to read every video's heap page
|
||||
# (681 MB on prod, a 2–6 s first open once something else had pushed it out of memory, E31 S4d).
|
||||
# (The Normal/Shorts/Live breakdown moved to the channel-detail page in R14, so it is no longer
|
||||
# (681 MB on prod, a 2–6 s first open once something else had pushed it out of memory).
|
||||
# (The Normal/Shorts/Live breakdown moved to the channel-detail page, so it is no longer
|
||||
# computed here — the manager table dropped its N/S/L column.)
|
||||
channel_ids = [c.id for _, c in rows]
|
||||
agg: dict[str, dict] = {}
|
||||
@@ -163,8 +163,8 @@ def discover_channels(
|
||||
can judge a channel BEFORE subscribing — but we do NOT pull its videos. Most-present
|
||||
channels come first; subscribing is a separate action."""
|
||||
# Resolve & cache the user's own channel id so it can be excluded — you don't subscribe
|
||||
# to yourself. Lazy + cached; needs OAuth, so demo / token-less users just skip it.
|
||||
if user.yt_channel_id is None and user.token is not None and not user.is_demo:
|
||||
# to yourself. Lazy + cached; needs OAuth, so token-less users just skip it.
|
||||
if user.yt_channel_id is None and user.token is not None:
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
@@ -192,8 +192,7 @@ def discover_channels(
|
||||
# (fresh subscriber counts/thumbnails carry through without a re-query). Only the title can
|
||||
# change, which affects the tie-break order → re-sort in Python instead of re-hitting the DB.
|
||||
need = [ch.id for ch, _, _ in rows if ch.details_synced_at is None][:DISCOVERY_ENRICH_CAP]
|
||||
# Skip the quota-spending enrich for the demo account (it just sees the stub metadata).
|
||||
if need and not user.is_demo:
|
||||
if need:
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
@@ -277,7 +276,7 @@ def _channel_detail_dict(
|
||||
"explored": explored,
|
||||
"blocked": blocked,
|
||||
"stored_videos": stored,
|
||||
# Per-type breakdown (Normal / Shorts / Live) — R14 moved it off the manager table onto the
|
||||
# Per-type breakdown (Normal / Shorts / Live) — moved off the manager table onto the
|
||||
# channel page. Same aggregation the manager list uses (count_normal = stored − shorts − live).
|
||||
"count_normal": count_normal,
|
||||
"count_short": count_short,
|
||||
@@ -311,12 +310,12 @@ def channel_detail(
|
||||
Lazily fills the About metadata (view count, join date, banner, links) the first time it's
|
||||
needed (a stub) and refreshes it once it is stale (`DETAILS_MAX_AGE`), which is how a channel nobody subscribes to (an explored one, outside the
|
||||
subscription import) gets a banner that still resolves. The enrich uses the API key (1 unit,
|
||||
cached after), so it's skipped for the quota-less demo account."""
|
||||
cached after)."""
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
# No `published_at is None` sentinel: a terminated channel never gets one, so it re-fetched on
|
||||
# every view; rows enriched before that field existed are long past DETAILS_MAX_AGE anyway.
|
||||
stale = details_stale(channel, datetime.now(timezone.utc))
|
||||
if stale and not user.is_demo:
|
||||
if stale:
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
@@ -350,7 +349,7 @@ def channel_detail(
|
||||
)
|
||||
blocked = _is_blocked(db, user, channel_id)
|
||||
# Stored total + the Normal/Shorts/Live split in one grouped pass (same filters as the manager
|
||||
# list), so the channel page can show the per-type breakdown R14 moved off the table.
|
||||
# list), so the channel page can show the per-type breakdown that moved off the table.
|
||||
n_all, n_short, n_live = db.execute(
|
||||
select(
|
||||
func.count(Video.id),
|
||||
@@ -389,7 +388,7 @@ def channel_detail(
|
||||
def explore_channel(
|
||||
channel_id: str,
|
||||
page_token: str | None = None,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Open an un-subscribed channel for browsing. Records the exploration (a per-user,
|
||||
@@ -397,7 +396,7 @@ def explore_channel(
|
||||
drives the later cleanup), flags the channel ephemeral unless someone organically follows
|
||||
it, then ingests one page of its uploads (newest-first) marked via_explore + enriched.
|
||||
Returns the next page token so the page can "load more". Guarded against the shared
|
||||
backfill quota reserve; blocked for the demo account (it spends shared quota)."""
|
||||
backfill quota reserve."""
|
||||
channel = get_or_404(db, Channel, channel_id, what="channel")
|
||||
if _is_blocked(db, user, channel_id):
|
||||
raise refused(403, "channel_blocked")
|
||||
@@ -493,10 +492,7 @@ def update_channel(
|
||||
sub.priority = body.priority
|
||||
if body.hidden is not None:
|
||||
sub.hidden = body.hidden
|
||||
# The demo account is SHARED, so it must not persist deep_requested (a quota-relevant flag) onto
|
||||
# the shared subscription — skip the write entirely for demo, which also keeps deep_turned_on
|
||||
# False so the immediate backfill below can never fire for it.
|
||||
if body.deep_requested is not None and not user.is_demo:
|
||||
if body.deep_requested is not None:
|
||||
# Opt this channel into (or out of) full-history backfill. The deep scheduler
|
||||
# picks the flag up on its next run; turning it off won't undo already-fetched
|
||||
# videos, it just stops further deep paging if nobody else still wants it.
|
||||
@@ -507,7 +503,7 @@ def update_channel(
|
||||
|
||||
# Opting a channel into full history should give an immediate feed: if we haven't even
|
||||
# fetched its recent uploads yet, do that one channel now (cheap) instead of waiting for
|
||||
# the scheduler. (deep_turned_on is only ever True for a non-demo user — see the guard above.)
|
||||
# the scheduler.
|
||||
if deep_turned_on and channel is not None and channel.recent_synced_at is None:
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
||||
run_recent_backfill(db, [channel], max_channels=1)
|
||||
|
||||
@@ -139,7 +139,7 @@ def test_issue_tracker(
|
||||
_: User = Depends(admin_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Check the configured issue tracker (E16): the repository answers with this token, takes
|
||||
"""Check the configured issue tracker: the repository answers with this token, takes
|
||||
issues, and is PRIVATE — the one it must be, since an issue carries its reporter's log."""
|
||||
try:
|
||||
return {"ok": True, **gitea.check(db)}
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
"""Debug sessions (E16): Start → reproduce → Stop, the screenshots, and the evidence under review.
|
||||
"""Debug sessions: Start → reproduce → Stop, the screenshots, and the evidence under review.
|
||||
|
||||
The session itself — what tags a request, its cap, its level — is `app/debugsession.py`. These routes
|
||||
are the user's handle on their OWN sessions only: another user's id 404s (`owned_or_404`), including
|
||||
for an admin, who reads everyone's lines in the diagnostics console instead. The shared demo account
|
||||
cannot record: its log would interleave every visitor under one account.
|
||||
for an admin, who reads everyone's lines in the diagnostics console instead.
|
||||
|
||||
Screenshots arrive as a raw image body (`image/png`, `image/jpeg`, `image/webp`), not a multipart
|
||||
form — one image per request, capped while it arrives, and kept only if its bytes really are one of
|
||||
@@ -21,7 +20,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, debugsession, logs, logsink
|
||||
from app.audit import AuditAction
|
||||
from app.auth import require_human
|
||||
from app.auth import current_user
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.issues import bundle, gitea
|
||||
from app.ratelimit import RateLimiter
|
||||
@@ -51,7 +50,7 @@ _new_sessions = RateLimiter(max_events=20, window_seconds=3600)
|
||||
@router.post("")
|
||||
def start_session(
|
||||
body: StartIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Start recording, or join the recording already running (another tab started it)."""
|
||||
@@ -79,7 +78,7 @@ def start_session(
|
||||
|
||||
@router.post("/{sid}/stop")
|
||||
def stop_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
sid: str, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
if session.ended_at is None:
|
||||
@@ -93,7 +92,7 @@ def stop_session(
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_sessions(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
def list_sessions(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
sessions = db.scalars(
|
||||
select(DebugSession)
|
||||
.where(DebugSession.user_id == user.id)
|
||||
@@ -105,14 +104,14 @@ def list_sessions(user: User = Depends(require_human), db: Session = Depends(get
|
||||
|
||||
@router.get("/{sid}")
|
||||
def get_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
sid: str, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
return _view(db, owned_or_404(db, DebugSession, sid, user, what="debug_session"))
|
||||
|
||||
|
||||
@router.get("/{sid}/events")
|
||||
def session_events(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
sid: str, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""The rows an issue from this session would carry — browser and server, in time order."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
@@ -141,7 +140,7 @@ class DraftIn(BaseModel):
|
||||
def save_draft(
|
||||
sid: str,
|
||||
body: DraftIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
@@ -162,7 +161,7 @@ def save_draft(
|
||||
|
||||
@router.get("/{sid}/export")
|
||||
def export_session(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
sid: str, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
"""The issue as a zip — the text, the two logs and the screenshots — for an instance without an
|
||||
issue tracker, or for sending it by hand. Built from the SAVED draft: what the composer shows is
|
||||
@@ -182,7 +181,7 @@ def export_session(
|
||||
)
|
||||
|
||||
|
||||
# --- the issue tracker (S4) -----------------------------------------------------------------
|
||||
# --- the issue tracker ----------------------------------------------------------------------
|
||||
# Issues a user may open per hour — counted once the tracker is ASKED to open one (an answer that
|
||||
# goes missing may still have created it, so an attempt cannot be given back); finishing a half-sent
|
||||
# one, or an attempt refused before the tracker was asked, is not counted. The composer sends one at
|
||||
@@ -208,9 +207,7 @@ def _issues_rate_limited() -> HTTPException:
|
||||
|
||||
|
||||
@router.post("/{sid}/issue")
|
||||
def send_issue(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
def send_issue(sid: str, user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""Send the session's issue to the tracker — from the SAVED draft, as the export is built — or
|
||||
finish a send that stopped half-way (the issue exists; only what is missing is sent)."""
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
@@ -302,7 +299,7 @@ class PreviewIn(BaseModel):
|
||||
def preview_issue(
|
||||
sid: str,
|
||||
body: PreviewIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""The body as the tracker will render it. Gitea sanitises its own output; the composer still
|
||||
@@ -323,14 +320,14 @@ def _shot_view(shot: DebugScreenshot) -> dict:
|
||||
|
||||
@router.get("/{sid}/screenshots")
|
||||
def list_screenshots(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
sid: str, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
return {"items": [_shot_view(s) for s in debugsession.screenshots(db, session)]}
|
||||
|
||||
|
||||
def _upload_target(
|
||||
sid: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
sid: str, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> DebugSession:
|
||||
return owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
|
||||
@@ -393,7 +390,7 @@ def _screenshot_of(db: Session, session: DebugSession, shot_id: int) -> DebugScr
|
||||
|
||||
@router.get("/{sid}/screenshots/{shot_id}")
|
||||
def get_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
sid: str, shot_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> Response:
|
||||
shot = _session_screenshot(db, sid, shot_id, user)
|
||||
# `nosniff`: the type was checked against the bytes at upload, and the browser must not guess.
|
||||
@@ -406,7 +403,7 @@ def get_screenshot(
|
||||
|
||||
@router.delete("/{sid}/screenshots/{shot_id}")
|
||||
def delete_screenshot(
|
||||
sid: str, shot_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
sid: str, shot_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
session = owned_or_404(db, DebugSession, sid, user, what="debug_session")
|
||||
shot = _screenshot_of(db, session, shot_id)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The browser's log, sent while its user has a raised diagnostics level (E22 S5).
|
||||
"""The browser's log, sent while its user has a raised diagnostics level.
|
||||
|
||||
The client keeps its log in memory (frontend `lib/log.ts`). Only after the user consented to Részletes
|
||||
or Teljes in Settings → Diagnosztika does it stream that log here, in batches, and each batch lands in
|
||||
@@ -14,7 +14,7 @@ Nothing from the body is trusted as-is:
|
||||
modified client would not;
|
||||
* an event below the consented level is dropped (a Részletes consent does not store what only
|
||||
Teljes may carry), and so is the whole batch once the level has lapsed — where an open debug
|
||||
session the request names counts as consent to ITS level (E16), and only its own events keep
|
||||
session the request names counts as consent to ITS level, and only its own events keep
|
||||
its tag;
|
||||
* the timestamp is clamped to the consent window, so a row cannot dodge the retention job by
|
||||
claiming to come from the future.
|
||||
@@ -63,7 +63,7 @@ class ClientEvent(BaseModel):
|
||||
attrs: dict[str, Any] | None = None
|
||||
request_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9._-]{8,64}$")
|
||||
tab_id: str | None = Field(default=None, pattern=r"^[A-Za-z0-9_-]{1,32}$")
|
||||
# The debug session the event was captured in (E16); kept only when it is the one this request
|
||||
# The debug session the event was captured in; kept only when it is the one this request
|
||||
# names in `X-Debug-Session` and the auth dependency found it the caller's own and open.
|
||||
debug_session_id: str | None = Field(default=None, pattern=r"^[0-9a-f]{16}$")
|
||||
version: str | None = Field(default=None, max_length=64)
|
||||
@@ -138,7 +138,7 @@ def ingest_events(
|
||||
body: ClientEventsIn = Depends(_parsed_events),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
# A debug session the request names raises the level for its own events (E16): starting one is
|
||||
# A debug session the request names raises the level for its own events: starting one is
|
||||
# the consent to that level for its duration, whatever the user's own level is.
|
||||
session = debugsession.from_header(db, user, request.headers.get(debugsession.HEADER))
|
||||
level, _ = debugsession.level_for(user, session)
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""E24 · the Home surface's HTTP shell.
|
||||
"""The Home surface's HTTP shell.
|
||||
|
||||
Thin by construction: every decision — what counts as in progress, what fits a time budget, which
|
||||
tier the hero comes from — lives in `app/discover/`, so the scheduler, a future CLI or a test can ask
|
||||
the same questions without a request. This file validates, gates and serializes.
|
||||
|
||||
The consolidated `GET /api/discover` mirrors the Plex Discover route's one-payload shape (E5 S4): a
|
||||
The consolidated `GET /api/discover` mirrors the Plex Discover route's one-payload shape: a
|
||||
landing page that fires four requests to answer one question would spend its whole budget on the
|
||||
thing it exists to remove.
|
||||
"""
|
||||
@@ -17,7 +17,7 @@ from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user, require_human
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.discover import hero as hero_mod
|
||||
from app.discover import pins, rails
|
||||
@@ -72,7 +72,7 @@ def get_discover(
|
||||
# never reaches SQL — `hero.pick` uses it only as set membership over an already-bounded candidate
|
||||
# pool (the resume rail's `RAIL_LIMIT` rows, the shelf, `SURPRISE_DRAW` surprise draws) — so there is
|
||||
# no scan for a crafted URL to unbound, and the set is built in full before this slice anyway. The
|
||||
# comment that claimed otherwise was corrected in round 2; the request line's own length is what
|
||||
# comment that claimed otherwise was later corrected; the request line's own length is what
|
||||
# actually bounds a hostile `exclude`.
|
||||
MAX_EXCLUDE = 32
|
||||
|
||||
@@ -127,15 +127,14 @@ def list_pins(
|
||||
@router.post("/pins")
|
||||
def add_pin(
|
||||
body: PinIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Pin something to the comfort shelf.
|
||||
|
||||
`require_human`, like every other personal-state write: the shared demo account must not be able
|
||||
to curate a communal shelf. Re-pinning something already on the shelf is a no-op that returns the
|
||||
existing pin rather than a 409 — the gesture is "make sure this is on my shelf", and the unique
|
||||
constraint is the mechanism, not the message."""
|
||||
Re-pinning something already on the shelf is a no-op that returns the existing pin rather than
|
||||
a 409 — the gesture is "make sure this is on my shelf", and the unique constraint is the
|
||||
mechanism, not the message."""
|
||||
resolved = pins.resolve_ref(db, user, body.kind, body.ref)
|
||||
if resolved is None:
|
||||
raise not_found("item")
|
||||
@@ -178,7 +177,7 @@ def add_pin(
|
||||
@router.delete("/pins/{pin_id}")
|
||||
def remove_pin(
|
||||
pin_id: int,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
pin = owned_or_404(db, ComfortPin, pin_id, user, what="pin")
|
||||
@@ -190,7 +189,7 @@ def remove_pin(
|
||||
@router.post("/pins/reorder")
|
||||
def reorder_pins(
|
||||
body: ReorderIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Rewrite the shelf order from the list of pin ids the client shows.
|
||||
|
||||
+134
-68
@@ -1,15 +1,14 @@
|
||||
"""Download center HTTP API — the per-user surface over the worker/cache/quota layers.
|
||||
|
||||
All routes are behind `require_human` (the shared demo account can't spend server disk / needs a
|
||||
real identity). Admin routes add `admin_user`. Live job state is polled by the frontend (the
|
||||
worker is a separate process); this module never downloads — it enqueues, manages job state,
|
||||
serves finished files (range-aware, with the user's custom display name), and shares them.
|
||||
All routes are behind `current_user`; admin routes add `admin_user`. Live job state is polled by the
|
||||
frontend (the worker is a separate process); this module never downloads — it enqueues, manages job
|
||||
state, serves finished files (range-aware, with the user's custom display name), and shares them.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from collections.abc import AsyncIterator
|
||||
from collections.abc import AsyncIterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
@@ -18,13 +17,14 @@ from urllib.parse import parse_qs, unquote, urlparse
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated
|
||||
from app import audit, sysconfig
|
||||
from app.auth import admin_user, require_human
|
||||
from app.auth import admin_user, current_user
|
||||
from app.config import settings
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.downloads import credentials as dlcreds
|
||||
@@ -33,17 +33,26 @@ from app.downloads import links as linksmod
|
||||
from app.downloads import quota, service, storage
|
||||
from app.downloads import upload as uploadmod
|
||||
from app.downloads import urlguard
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
DownloadProfile,
|
||||
DownloadQuota,
|
||||
DownloadShare,
|
||||
LibraryState,
|
||||
MediaAsset,
|
||||
User,
|
||||
display_text,
|
||||
)
|
||||
from app.routes._common import capped_stream_to, get_or_404, not_found, owned_or_404
|
||||
from app.routes._common import (
|
||||
ProgressIn,
|
||||
capped_stream_to,
|
||||
get_or_404,
|
||||
kept_position,
|
||||
not_found,
|
||||
owned_or_404,
|
||||
)
|
||||
from app.security import hash_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
@@ -132,8 +141,8 @@ def _serialize(job: DownloadJob, asset: MediaAsset | None) -> dict:
|
||||
"job_kind": job.job_kind,
|
||||
"source_job_id": job.source_job_id,
|
||||
"edit_spec": job.edit_spec,
|
||||
"can_download": ready and job.status == "done",
|
||||
"expired": job.asset_id is None and job.status == "done",
|
||||
"can_download": ready and job.status == JobStatus.DONE,
|
||||
"expired": job.asset_id is None and job.status == JobStatus.DONE,
|
||||
"asset": None
|
||||
if asset is None
|
||||
else {
|
||||
@@ -165,6 +174,19 @@ def _assets_for(db: Session, jobs: list[DownloadJob]) -> dict[int, MediaAsset]:
|
||||
return {a.id: a for a in rows}
|
||||
|
||||
|
||||
def _positions_for(db: Session, user: User, jobs: Sequence[DownloadJob]) -> dict[int, int]:
|
||||
"""`user`'s resume point in each of `jobs` that has one (the library player's `library_states`)."""
|
||||
ids = [j.id for j in jobs]
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(
|
||||
select(LibraryState.job_id, LibraryState.position_seconds).where(
|
||||
LibraryState.user_id == user.id, LibraryState.job_id.in_(ids)
|
||||
)
|
||||
).all()
|
||||
return {job_id: pos for job_id, pos in rows}
|
||||
|
||||
|
||||
def _serialize_job(db: Session, job: DownloadJob) -> dict:
|
||||
"""Resolve a job's cache asset (if any) and serialize the pair — the resolve-then-serialize step
|
||||
every single-job handler shares."""
|
||||
@@ -212,7 +234,7 @@ def _serialize_profile(p: DownloadProfile) -> dict:
|
||||
|
||||
|
||||
@router.get("/profiles")
|
||||
def list_profiles(user: User = Depends(require_human), db: Session = Depends(get_db)) -> list[dict]:
|
||||
def list_profiles(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
rows = (
|
||||
db.execute(
|
||||
select(DownloadProfile)
|
||||
@@ -234,7 +256,7 @@ class CreateProfileIn(BaseModel):
|
||||
|
||||
@router.post("/profiles")
|
||||
def create_profile(
|
||||
body: CreateProfileIn, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
body: CreateProfileIn, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
name = body.name.strip()
|
||||
spec = body.spec
|
||||
@@ -262,7 +284,7 @@ class UpdateProfileIn(BaseModel):
|
||||
def update_profile(
|
||||
profile_id: int,
|
||||
body: UpdateProfileIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
@@ -284,7 +306,7 @@ def update_profile(
|
||||
|
||||
@router.delete("/profiles/{profile_id}")
|
||||
def delete_profile(
|
||||
profile_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
profile_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
p = db.get(DownloadProfile, profile_id)
|
||||
if p is None or p.is_builtin or p.user_id != user.id:
|
||||
@@ -300,7 +322,7 @@ def delete_profile(
|
||||
|
||||
|
||||
@router.get("/credentials")
|
||||
def list_credentials(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
def list_credentials(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
return {"hosts": dlcreds.status_list(db, user.id)}
|
||||
|
||||
|
||||
@@ -312,7 +334,7 @@ class SetCredentialIn(BaseModel):
|
||||
def set_credential(
|
||||
host_key: str,
|
||||
body: SetCredentialIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Store (encrypted) this user's cookies.txt for one supported host. Write-only: the value is
|
||||
@@ -337,7 +359,7 @@ def set_credential(
|
||||
|
||||
@router.delete("/credentials/{host_key}")
|
||||
def delete_credential(
|
||||
host_key: str, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
host_key: str, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
if not dlcreds.is_supported_host(host_key):
|
||||
raise not_found("host")
|
||||
@@ -388,7 +410,7 @@ class EnqueueDownloadIn(BaseModel):
|
||||
|
||||
@router.post("")
|
||||
def enqueue_download(
|
||||
body: EnqueueDownloadIn, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
body: EnqueueDownloadIn, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
source_kind, source_ref = resolve_source(body.source)
|
||||
# SEC-12: a non-YouTube source is a raw URL the worker will fetch server-side, so refuse the
|
||||
@@ -413,8 +435,8 @@ def enqueue_download(
|
||||
|
||||
# The upload's filename and display name travel as HEADERS, not as multipart parts. That is not a
|
||||
# style choice: a DECLARED body parameter (`file: UploadFile = File(...)`) is read and spooled by
|
||||
# FastAPI *before the first dependency runs*, so `require_human` could not gate it and an anonymous
|
||||
# caller could make the process write the whole per-file allowance to temp, per request (P1; SEC-20
|
||||
# FastAPI *before the first dependency runs*, so the auth dependency could not gate it and an anonymous
|
||||
# caller could make the process write the whole per-file allowance to temp, per request (SEC-20
|
||||
# bounded the number but could not stop the writing). Reading the body ourselves, from a dependency
|
||||
# declared AFTER the auth one, is the only shape that closes it — the same one
|
||||
# `POST /api/debug-sessions/{sid}/screenshots` uses, and `test_body_limit.py` now enforces it for
|
||||
@@ -454,13 +476,13 @@ def _header_text(request: Request, name: str) -> str:
|
||||
|
||||
async def spool_upload(
|
||||
request: Request,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> AsyncIterator[SpooledUpload]:
|
||||
"""Stream the request body to a staging temp file, enforcing the per-file cap as it arrives.
|
||||
|
||||
`user` is taken but unused: it is the whole point. Dependencies resolve in declaration order, so
|
||||
naming `require_human` here makes the auth check a precondition of reading the body at all — a
|
||||
naming `current_user` here makes the auth check a precondition of reading the body at all — a
|
||||
stranger is refused before a byte moves. The `yield` shape owns the temp file's life: ingest
|
||||
MOVES it into place on success (the unlink is then a no-op), and on any other exit — cap hit,
|
||||
empty, quota, probe or move error, a client that vanished mid-upload — it is removed."""
|
||||
@@ -499,11 +521,12 @@ def upload_download(
|
||||
upload: SpooledUpload = Depends(spool_upload),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Ingest a local file into the caller's library (E7 S5). Any file type is accepted; media
|
||||
"""Ingest a local file into the caller's library. Any file type is accepted; media
|
||||
(video/audio) gets the full function set, a generic file is download-only. The body IS the file
|
||||
(see the header note above); `spool_upload` has already authenticated the caller and streamed it
|
||||
to a temp path under the download volume, so what is left here is the non-yt-dlp ingest which
|
||||
probes + places it and returns an already-`done` job.
|
||||
probes + places it and returns its job — `done`, or `queued` for the worker's phone-safe
|
||||
conversion (`downloads.upload`).
|
||||
|
||||
A sync (threadpool) handler on purpose: the ingest does blocking work — an ffprobe of up to 30s
|
||||
plus sync DB writes — so it must NOT run on the event loop. The streaming half is async because
|
||||
@@ -532,7 +555,7 @@ class EnqueueEditIn(BaseModel):
|
||||
|
||||
@router.post("/edit")
|
||||
def enqueue_edit(
|
||||
body: EnqueueEditIn, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
body: EnqueueEditIn, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Queue a phase-2 editor derivative (trim/crop) of a download this user can access.
|
||||
|
||||
@@ -540,7 +563,7 @@ def enqueue_edit(
|
||||
clip is a NEW per-user derivative in the editor's own library (counts against their quota); the
|
||||
source file is only read, never modified, so editing a shared video is safe."""
|
||||
source_job = _accessible_job(db, user, body.source_job_id)
|
||||
if source_job.status != "done" or source_job.asset_id is None:
|
||||
if source_job.status != JobStatus.DONE or source_job.asset_id is None:
|
||||
raise refused(409, "edit_unfinished")
|
||||
spec = body.edit_spec
|
||||
display_name = display_text(body.display_name)
|
||||
@@ -560,9 +583,7 @@ def _quota_detail(e: quota.QuotaExceeded) -> dict:
|
||||
|
||||
|
||||
@router.get("")
|
||||
def list_downloads(
|
||||
user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
def list_downloads(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
jobs = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
@@ -574,11 +595,18 @@ def list_downloads(
|
||||
.all()
|
||||
)
|
||||
assets = _assets_for(db, jobs)
|
||||
return [_serialize(j, assets.get(j.asset_id)) for j in jobs]
|
||||
positions = _positions_for(db, user, jobs)
|
||||
return [
|
||||
{
|
||||
**_serialize(j, assets.get(j.asset_id) if j.asset_id is not None else None),
|
||||
"position_seconds": positions.get(j.id, 0),
|
||||
}
|
||||
for j in jobs
|
||||
]
|
||||
|
||||
|
||||
@router.get("/usage")
|
||||
def my_usage(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
def my_usage(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""The caller's download quota, plus the per-file upload cap.
|
||||
|
||||
The cap rides here rather than in `quota.usage` because it is not a quota value — it is a
|
||||
@@ -596,17 +624,22 @@ def my_usage(user: User = Depends(require_human), db: Session = Depends(get_db))
|
||||
|
||||
|
||||
# Priority when a source has several jobs (e.g. a done one plus a fresh queued one).
|
||||
_INDEX_RANK = {"done": 4, "running": 3, "paused": 2, "queued": 1}
|
||||
_INDEX_RANK: dict[str, int] = {
|
||||
JobStatus.DONE: 4,
|
||||
JobStatus.RUNNING: 3,
|
||||
JobStatus.PAUSED: 2,
|
||||
JobStatus.QUEUED: 1,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/index")
|
||||
def my_download_index(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
def my_download_index(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""source_ref -> best active status, for the feed's per-card "downloaded / queued" badge.
|
||||
Small + cheap so the feed can poll it without loading the full job list."""
|
||||
rows = db.execute(
|
||||
select(DownloadJob.source_ref, DownloadJob.status).where(
|
||||
DownloadJob.user_id == user.id,
|
||||
DownloadJob.status.in_(("queued", "running", "paused", "done")),
|
||||
DownloadJob.status.in_(tuple(_INDEX_RANK)),
|
||||
)
|
||||
).all()
|
||||
out: dict[str, str] = {}
|
||||
@@ -617,9 +650,7 @@ def my_download_index(user: User = Depends(require_human), db: Session = Depends
|
||||
|
||||
|
||||
@router.get("/shared")
|
||||
def shared_with_me(
|
||||
user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
def shared_with_me(user: User = Depends(current_user), db: Session = Depends(get_db)) -> list[dict]:
|
||||
jobs = (
|
||||
db.execute(
|
||||
select(DownloadJob)
|
||||
@@ -632,10 +663,12 @@ def shared_with_me(
|
||||
.all()
|
||||
)
|
||||
assets = _assets_for(db, jobs)
|
||||
positions = _positions_for(db, user, jobs)
|
||||
out = []
|
||||
for j in jobs:
|
||||
data = _serialize(j, assets.get(j.asset_id))
|
||||
data["shared"] = True
|
||||
data["position_seconds"] = positions.get(j.id, 0)
|
||||
out.append(data)
|
||||
return out
|
||||
|
||||
@@ -679,7 +712,7 @@ class UpdateDownloadMetaIn(BaseModel):
|
||||
def update_download_meta(
|
||||
job_id: int,
|
||||
body: UpdateDownloadMetaIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Update a download's display metadata: title, channel name + link, and extra reference URLs.
|
||||
@@ -702,7 +735,7 @@ def update_download_meta(
|
||||
|
||||
@router.post("/{job_id}/pause")
|
||||
def pause_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.pause(db, job)
|
||||
@@ -711,7 +744,7 @@ def pause_download(
|
||||
|
||||
@router.post("/{job_id}/resume")
|
||||
def resume_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.resume(db, job)
|
||||
@@ -720,7 +753,7 @@ def resume_download(
|
||||
|
||||
@router.post("/{job_id}/redownload")
|
||||
def redownload_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Force a fresh re-download of a finished (or failed) job, rebuilding the file + Plex tree.
|
||||
|
||||
@@ -739,7 +772,7 @@ def redownload_download(
|
||||
# Neither has a re-fetchable source: an editor clip is derived locally, an upload IS the
|
||||
# source. Re-running yt-dlp against source_kind='upload'/'edit' would just fail.
|
||||
raise refused(409, "not_redownloadable")
|
||||
if job.status in ("queued", "running", "paused"):
|
||||
if job.status in (JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED):
|
||||
raise refused(409, "still_downloading")
|
||||
service.redownload(db, job)
|
||||
return _serialize_job(db, job)
|
||||
@@ -747,7 +780,7 @@ def redownload_download(
|
||||
|
||||
@router.post("/{job_id}/cancel")
|
||||
def cancel_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.cancel(db, job)
|
||||
@@ -756,7 +789,7 @@ def cancel_download(
|
||||
|
||||
@router.delete("/{job_id}")
|
||||
def delete_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job_for_update(db, user, job_id)
|
||||
service.delete(db, job)
|
||||
@@ -786,7 +819,7 @@ def _accessible_job(db: Session, user: User, job_id: int) -> DownloadJob:
|
||||
def download_file(
|
||||
job_id: int,
|
||||
request: Request,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
job = _accessible_job(db, user, job_id)
|
||||
@@ -823,9 +856,42 @@ def download_file(
|
||||
# --- editor filmstrip (scrub track) --------------------------------------------------------
|
||||
|
||||
|
||||
@router.post("/{job_id}/progress")
|
||||
def set_library_progress(
|
||||
job_id: int,
|
||||
body: ProgressIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Checkpoint the library player's resume point in a download the caller can play (their own, or
|
||||
one shared with them). A position too early or too close to the end clears it instead."""
|
||||
_accessible_job(db, user, job_id)
|
||||
position = kept_position(body)
|
||||
mine = (LibraryState.user_id == user.id) & (LibraryState.job_id == job_id)
|
||||
if not position:
|
||||
# A statement, not a loaded row's delete: the periodic checkpoint and the pagehide beacon of
|
||||
# one stop may both clear it, and the second then deletes nothing rather than failing.
|
||||
db.execute(delete(LibraryState).where(mine))
|
||||
db.commit()
|
||||
return {"ok": True, "job_id": job_id, "position_seconds": 0}
|
||||
now = datetime.now(timezone.utc)
|
||||
# One row per (user, job) even when two checkpoints race (the same pair): an upsert, not a
|
||||
# read-then-insert that the second would fail on the unique key.
|
||||
db.execute(
|
||||
pg_insert(LibraryState)
|
||||
.values(user_id=user.id, job_id=job_id, position_seconds=position, progress_updated_at=now)
|
||||
.on_conflict_do_update(
|
||||
constraint="uq_library_user_job",
|
||||
set_={"position_seconds": position, "progress_updated_at": now},
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "job_id": job_id, "position_seconds": position}
|
||||
|
||||
|
||||
@router.get("/{job_id}/storyboard")
|
||||
def storyboard_meta(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Filmstrip geometry for the editor's scrub track, generating the sprite on first use.
|
||||
|
||||
@@ -850,7 +916,7 @@ def storyboard_meta(
|
||||
|
||||
|
||||
@router.get("/{job_id}/poster.jpg")
|
||||
def poster_image(job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)):
|
||||
def poster_image(job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)):
|
||||
"""The locally-generated poster frame for a download whose source had no thumbnail."""
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
@@ -864,7 +930,7 @@ def poster_image(job_id: int, user: User = Depends(require_human), db: Session =
|
||||
|
||||
@router.get("/{job_id}/storyboard.jpg")
|
||||
def storyboard_image(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
):
|
||||
job = _accessible_job(db, user, job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job.asset_id else None
|
||||
@@ -887,11 +953,11 @@ class ShareDownloadIn(BaseModel):
|
||||
def share_download(
|
||||
job_id: int,
|
||||
body: ShareDownloadIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
if job.status != JobStatus.DONE or job.asset_id is None:
|
||||
raise refused(409, "share_unfinished")
|
||||
target = body.email.strip().lower()
|
||||
if not target:
|
||||
@@ -899,7 +965,7 @@ def share_download(
|
||||
recipient = db.execute(
|
||||
select(User).where(func.lower(User.email) == target)
|
||||
).scalar_one_or_none()
|
||||
# Uniform "no user" for a missing OR non-messageable (demo/suspended/deactivated) target, so
|
||||
# Uniform "no user" for a missing OR non-messageable (suspended/deactivated) target, so
|
||||
# sharing can't be used to probe which addresses exist or to reach a suspended account.
|
||||
if not is_messageable_user(recipient):
|
||||
raise not_found("recipient")
|
||||
@@ -919,7 +985,7 @@ def share_download(
|
||||
user_id=recipient.id,
|
||||
type="download_shared",
|
||||
title=(job.display_name or "A download") + " was shared with you",
|
||||
# The client names the notice in the reader's language from `data` (E30); the title is the
|
||||
# The client names the notice in the reader's language from `data`; the title is the
|
||||
# English fallback only.
|
||||
data={
|
||||
"kind": "download_shared",
|
||||
@@ -937,7 +1003,7 @@ def share_download(
|
||||
def unshare_download(
|
||||
job_id: int,
|
||||
email: str,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
owned_or_404(
|
||||
@@ -961,7 +1027,7 @@ def unshare_download(
|
||||
|
||||
@router.delete("/shared/{job_id}")
|
||||
def remove_shared_with_me(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""Remove a download that was shared WITH this user from their 'Shared with me' list. Deletes
|
||||
only the recipient's share grant — the owner's job and the physical file are untouched (this is
|
||||
@@ -982,10 +1048,10 @@ def remove_shared_with_me(
|
||||
|
||||
@router.get("/recipients")
|
||||
def share_recipients(
|
||||
user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
"""Registered users this user can share a download with (excludes self + any non-messageable
|
||||
account — demo, suspended, or deactivated — so the picker never leaks those addresses)."""
|
||||
account — suspended or deactivated — so the picker never leaks those addresses)."""
|
||||
rows = (
|
||||
db.execute(
|
||||
select(User)
|
||||
@@ -1021,7 +1087,7 @@ def _link_expiry(days: int | None) -> datetime | None:
|
||||
|
||||
@router.get("/{job_id}/links")
|
||||
def list_links(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
job_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
rows = (
|
||||
@@ -1038,7 +1104,7 @@ def list_links(
|
||||
|
||||
class CreateLinkIn(BaseModel):
|
||||
password: str = ""
|
||||
# D1 (E7 S3): a newly shared download is downloadable by default — the common intent is "let the
|
||||
# A newly shared download is downloadable by default — the common intent is "let the
|
||||
# recipient have the file", not stream-only. Forward-only: this only sets the default for links
|
||||
# minted from now on; existing links keep whatever `allow_download` they were created with (the
|
||||
# column default stays False, so no back-catalogue link silently flips). The owner can still
|
||||
@@ -1051,11 +1117,11 @@ class CreateLinkIn(BaseModel):
|
||||
def create_link(
|
||||
job_id: int,
|
||||
body: CreateLinkIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
job = owned_or_404(db, DownloadJob, job_id, user, what="download")
|
||||
if job.status != "done" or job.asset_id is None:
|
||||
if job.status != JobStatus.DONE or job.asset_id is None:
|
||||
raise refused(409, "share_unfinished")
|
||||
pw = body.password.strip()
|
||||
link = DownloadLink(
|
||||
@@ -1082,7 +1148,7 @@ class UpdateLinkIn(BaseModel):
|
||||
def update_link(
|
||||
link_id: int,
|
||||
body: UpdateLinkIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
link = _own_link(db, user, link_id)
|
||||
@@ -1103,7 +1169,7 @@ def update_link(
|
||||
|
||||
@router.delete("/links/{link_id}")
|
||||
def revoke_link(
|
||||
link_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
link_id: int, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
link = _own_link(db, user, link_id)
|
||||
db.delete(link)
|
||||
@@ -1142,10 +1208,10 @@ def admin_list_downloads(
|
||||
|
||||
@admin_router.get("/storage")
|
||||
def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_db)) -> dict:
|
||||
ready = db.execute(
|
||||
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(
|
||||
MediaAsset.status == "ready"
|
||||
)
|
||||
# Every file on disk (`quota.ON_DISK`): ready ones, and uploads waiting for or failed in their
|
||||
# conversion — the same disk the footprint and the GC cap count.
|
||||
on_disk = db.execute(
|
||||
select(func.count(), func.coalesce(func.sum(MediaAsset.size_bytes), 0)).where(quota.ON_DISK)
|
||||
).one()
|
||||
total_assets = db.execute(select(func.count()).select_from(MediaAsset)).scalar()
|
||||
per_user = db.execute(
|
||||
@@ -1158,8 +1224,8 @@ def admin_storage(admin: User = Depends(admin_user), db: Session = Depends(get_d
|
||||
for u in db.execute(select(User).where(User.id.in_([uid for uid, _ in per_user]))).scalars()
|
||||
}
|
||||
return {
|
||||
"ready_files": ready[0],
|
||||
"total_bytes": int(ready[1]),
|
||||
"files_on_disk": on_disk[0],
|
||||
"total_bytes": int(on_disk[1]),
|
||||
"total_assets": total_assets,
|
||||
"total_cap_bytes": sysconfig.get_int(db, "download_total_max_bytes"),
|
||||
"per_user": [
|
||||
|
||||
+13
-40
@@ -1,7 +1,7 @@
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -26,14 +26,19 @@ from app.feedquery import (
|
||||
windowed_relevance_rows,
|
||||
)
|
||||
from app.models import (
|
||||
POSITION_MAX_S,
|
||||
ChannelTag,
|
||||
Tag,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.routes._common import get_or_404, youtube_refusal, not_found
|
||||
from app.routes._common import (
|
||||
ProgressIn,
|
||||
get_or_404,
|
||||
kept_position,
|
||||
not_found,
|
||||
youtube_refusal,
|
||||
)
|
||||
from app.sync.videos import parse_iso8601_duration
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
@@ -44,14 +49,6 @@ router = APIRouter(prefix="/api", tags=["feed"])
|
||||
# "saved" used to be a status; it's now membership in the built-in Watch later playlist.
|
||||
VALID_STATES = {"new", "watched", "hidden"}
|
||||
|
||||
# Resume-position thresholds (mirror the client): positions below this are "didn't
|
||||
# really start" and within this of the end are "basically finished" — neither is worth
|
||||
# storing, so we clear the position instead (the video shows no progress bar).
|
||||
# NB: the Plex player intentionally uses a LARGER finish margin (30s) because it marks
|
||||
# long-form items watched, not just clears a resume bar — see routes/plex/playback.py.
|
||||
PROGRESS_MIN_SECONDS = 5
|
||||
FINISH_MARGIN_SECONDS = 10
|
||||
|
||||
|
||||
# Shared query parameters for /feed and /feed/count.
|
||||
def _feed_params(
|
||||
@@ -235,7 +232,7 @@ def get_feed_years(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Per-year video counts for the year control (item 8 / E8·S2): the quick recent-year chips and the
|
||||
"""Per-year video counts for the year control: the quick recent-year chips and the
|
||||
"more…" per-year-counts dialog. Every filter EXCEPT the publish-date narrowing applies, so picking a
|
||||
year re-queries the feed but not this list. Response: {"years": [{"year": <int>, "count": <int>}]},
|
||||
newest year first."""
|
||||
@@ -326,23 +323,10 @@ def clear_video_state(
|
||||
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
||||
|
||||
|
||||
class VideoProgressIn(BaseModel):
|
||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||
# one into a 500.
|
||||
model_config = ConfigDict(allow_inf_nan=False)
|
||||
|
||||
# Accept a float and truncate (below) — the player may post a raw currentTime (a float), which
|
||||
# the old `int(payload.get(...))` silently truncated. A plain `int` field would 422 that on the
|
||||
# 10s progress hot path, losing the resume position; `float` keeps the old tolerance.
|
||||
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
||||
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
|
||||
|
||||
@router.post("/videos/{video_id}/progress")
|
||||
def set_video_progress(
|
||||
video_id: str,
|
||||
body: VideoProgressIn,
|
||||
body: ProgressIn,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -350,22 +334,15 @@ def set_video_progress(
|
||||
Stores a per-user position on the video_states row without touching watch status, so a
|
||||
partially-watched video can render a progress bar and match the "in progress" filter.
|
||||
Trivially-early and near-finished positions clear the position rather than store it."""
|
||||
position = int(body.position_seconds)
|
||||
duration = int(body.duration_seconds)
|
||||
if position < 0:
|
||||
position = 0
|
||||
position = kept_position(body)
|
||||
|
||||
get_or_404(db, Video, video_id, what="video")
|
||||
|
||||
# Decide whether this position is worth keeping (else clear it).
|
||||
near_end = duration > 0 and position > duration - FINISH_MARGIN_SECONDS
|
||||
keep = position >= PROGRESS_MIN_SECONDS and not near_end
|
||||
|
||||
row = db.execute(
|
||||
select(VideoState).where(VideoState.user_id == user.id, VideoState.video_id == video_id)
|
||||
).scalar_one_or_none()
|
||||
|
||||
if not keep:
|
||||
if not position:
|
||||
# Nothing meaningful to store: drop a status-less row entirely, otherwise just
|
||||
# zero the position (a saved/hidden video keeps its status).
|
||||
if row is not None:
|
||||
@@ -419,11 +396,7 @@ def get_video_detail(
|
||||
"duration_seconds": v.duration_seconds,
|
||||
}
|
||||
|
||||
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit, so the
|
||||
# demo account can't reach it — it just sees "unknown" for a video we don't already store.
|
||||
if user.is_demo:
|
||||
raise not_found("video")
|
||||
|
||||
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit.
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP),
|
||||
|
||||
+12
-20
@@ -1,6 +1,6 @@
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, Request
|
||||
from pydantic import BaseModel, RootModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -40,7 +40,7 @@ def my_accounts(
|
||||
"""The accounts that have authenticated in this browser session (switchable without a
|
||||
new Google sign-in). The active one is flagged."""
|
||||
ids = request.session.get("account_ids") or [user.id]
|
||||
rows = {u.id: u for u in db.query(User).filter(User.id.in_(ids)).all()}
|
||||
rows = {u.id: u for u in db.execute(select(User).where(User.id.in_(ids))).scalars()}
|
||||
out = []
|
||||
for uid in ids: # preserve recency order from the session
|
||||
u = rows.get(uid)
|
||||
@@ -131,7 +131,6 @@ def get_me(
|
||||
"display_name": user.display_name,
|
||||
"avatar_url": user.avatar_url,
|
||||
"role": user.role,
|
||||
"is_demo": user.is_demo,
|
||||
"has_google": user.google_sub is not None,
|
||||
# The address Google answers with, when it DIFFERS from the account's own. Same string for
|
||||
# most people, and then this is None so no surface has to decide whether to show it — the
|
||||
@@ -141,13 +140,16 @@ def get_me(
|
||||
),
|
||||
"has_password": user.password_hash is not None,
|
||||
# The account's latest sign-in on any device — shown in Settings → Account, so a sign-in the
|
||||
# user does not recognise is something they can see (null: none since 0.88, or the demo).
|
||||
# user does not recognise is something they can see (null: none since 0.88).
|
||||
"last_login_at": iso(user.last_login_at),
|
||||
# Instance-wide: whether Google sign-in / YouTube connect is available at all (the UI hides
|
||||
# YouTube-access affordances and the onboarding nudge when it isn't configured).
|
||||
"google_enabled": google_enabled(db),
|
||||
# Whether the optional Plex module is enabled (UI shows the Plex feed source when so).
|
||||
"plex_enabled": sysconfig.get_bool(db, "plex_enabled"),
|
||||
# Which live-search source runs, so the search button can say what a search costs: "scrape"
|
||||
# spends no API quota, "api" spends 100 units a page.
|
||||
"search_source": sysconfig.get_str(db, "search_source"),
|
||||
"can_read": has_read_scope(user),
|
||||
"can_write": has_write_scope(user),
|
||||
"pending_invites": pending_invites,
|
||||
@@ -156,12 +158,12 @@ def get_me(
|
||||
"pending_activations": pending_activations,
|
||||
"email_configured": email_configured,
|
||||
"preferences": user.preferences or {},
|
||||
# The consented diagnostics level in force and when it lapses (E22 S5). The client applies
|
||||
# The consented diagnostics level in force and when it lapses. The client applies
|
||||
# it to its own log and streams that log while it is raised.
|
||||
"diagnostics": diagnostics.state(db, user),
|
||||
# The live debug session, if one is recording (E16) — a reload resumes it.
|
||||
# The live debug session, if one is recording — a reload resumes it.
|
||||
"debug_session": _live_session(db, user),
|
||||
# Where a bug report goes (E16 S4): the tracker's repository, or None — download only.
|
||||
# Where a bug report goes: the tracker's repository, or None — download only.
|
||||
"issues": gitea.target(db),
|
||||
}
|
||||
|
||||
@@ -181,15 +183,8 @@ def delete_my_account(
|
||||
"""GDPR self-service erasure: permanently delete the signed-in account and ALL its personal
|
||||
data — OAuth tokens, subscriptions, tags, watch/save/hide states, playlists, notifications —
|
||||
which all cascade on the users row. Quota-audit events are kept but anonymised (FK SET NULL).
|
||||
The shared demo account can't be deleted, and the last remaining admin can't delete itself
|
||||
(that would lock everyone out of the admin surfaces)."""
|
||||
# Structured, like the other three demo refusals: `req` leaves 403 to the caller and
|
||||
# `notifyActionError` discards a plain-string detail in favour of the caller's translated
|
||||
# fallback, so a hand-written English sentence here reaches a Hungarian UI as a generic
|
||||
# "couldn't do that" — or, on this affordance, as nothing at all. `reason` distinguishes it
|
||||
# from the other sites' bare `{code: "demo"}` so the UI can say what specifically is refused.
|
||||
if user.is_demo:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "delete_account"})
|
||||
The last remaining admin can't delete itself (that would lock everyone out of the admin
|
||||
surfaces)."""
|
||||
if user.role == "admin" and count_admins(db) <= 1:
|
||||
raise refused(400, "only_admin_delete")
|
||||
user_id = user.id
|
||||
@@ -239,10 +234,7 @@ def update_diagnostics(
|
||||
) -> dict:
|
||||
"""Raise the diagnostics level (the user consented in Settings → Diagnosztika) or return to the
|
||||
base. The expiry is the server's: a raised level always lapses after
|
||||
`diagnostics.ELEVATION_HOURS`, whatever the client asks. The shared demo account cannot raise
|
||||
it — its log would interleave every visitor's session under one account."""
|
||||
if user.is_demo and body.level != diagnostics.BASE:
|
||||
raise HTTPException(status_code=403, detail={"code": "demo", "reason": "diagnostics"})
|
||||
`diagnostics.ELEVATION_HOURS`, whatever the client asks."""
|
||||
diagnostics.set_level(user, body.level)
|
||||
db.commit()
|
||||
# The rest of THIS request, and the process's elevation registry, follow the new level now.
|
||||
|
||||
@@ -1,479 +0,0 @@
|
||||
"""Direct messaging (notification module, phase 2): end-to-end encrypted user-to-user chat
|
||||
plus server-authored system messages, with live WebSocket delivery.
|
||||
|
||||
E2EE model: every private message is encrypted IN THE BROWSER under a key both parties derive
|
||||
from ECDH(their private key, the other's public key). The server only ever stores ciphertext +
|
||||
iv and acts as a key directory (public keys) and an opaque blob store (each user's private key,
|
||||
wrapped client-side with a passphrase the server never sees). So not even an admin can read a
|
||||
user-to-user conversation. `kind="system"` messages (the welcome; future announcements) are
|
||||
plain server boilerplate — non-private, readable without any key setup, shown as a "Siftlode"
|
||||
conversation.
|
||||
|
||||
Every endpoint is gated by `require_human` (the shared demo account can't message). Metadata
|
||||
(who messaged whom, when, read state) is NOT encrypted — standard for E2EE and needed for
|
||||
routing and unread counts.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, WebSocket, WebSocketDisconnect
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import and_, case, delete, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.mailtext import lang_for_user
|
||||
from app.collation import collated
|
||||
from app.auth import require_human, resolved_user_id
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import Message, MessageKey, User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.realtime import manager
|
||||
from app.security import verify_password
|
||||
from app.userscope import is_messageable_user, messageable_clauses
|
||||
from app.utils import iso
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
|
||||
router = APIRouter(prefix="/api/messages", tags=["messages"])
|
||||
|
||||
MAX_CIPHERTEXT = 8000 # base64 of a generous plaintext + AES-GCM overhead
|
||||
SYSTEM_PARTNER_ID = 0 # synthetic conversation partner for system messages
|
||||
SYSTEM_NAME = "Siftlode"
|
||||
_send_limiter = RateLimiter(max_events=30, window_seconds=60)
|
||||
_reset_key_limiter = RateLimiter(max_events=5, window_seconds=300)
|
||||
|
||||
# The welcome is rendered server-side at creation time, so it carries its own translations
|
||||
# (one per supported UI language). Kept short and non-private (every new user gets the same text).
|
||||
WELCOME = {
|
||||
"en": (
|
||||
"Welcome to Siftlode! 👋\n\n"
|
||||
"This is your private message inbox. Conversations with other members are end-to-end "
|
||||
"encrypted — only you and the person you're chatting with can read them, not even an "
|
||||
"admin. To start messaging someone, set up secure messaging with a passphrase first.\n\n"
|
||||
"Enjoy your Feed!"
|
||||
),
|
||||
"hu": (
|
||||
"Üdv a Siftlode-on! 👋\n\n"
|
||||
"Ez a privát üzenet-postafiókod. A többi taggal folytatott beszélgetések végpontig "
|
||||
"titkosítottak — csak te és a beszélgetőpartnered olvashatja őket, még az admin sem. "
|
||||
"Ha üzenni szeretnél valakinek, előbb állítsd be a biztonságos üzenetküldést egy "
|
||||
"jelszóval.\n\n"
|
||||
"Jó böngészést!"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class KeySetupIn(BaseModel):
|
||||
public_key: str
|
||||
wrapped_private_key: str
|
||||
salt: str
|
||||
wrap_iv: str
|
||||
key_check: str
|
||||
|
||||
|
||||
class KeyResetIn(BaseModel):
|
||||
# Present only for password accounts; a Google-only account has no password to re-verify.
|
||||
current_password: str | None = None
|
||||
|
||||
|
||||
class SendIn(BaseModel):
|
||||
recipient_id: int
|
||||
ciphertext: str
|
||||
iv: str
|
||||
|
||||
|
||||
def _user_label(u: User) -> str:
|
||||
return u.display_name or u.email.split("@")[0]
|
||||
|
||||
|
||||
def _serialize_user(u: User) -> dict:
|
||||
return {"id": u.id, "name": _user_label(u), "avatar_url": u.avatar_url}
|
||||
|
||||
|
||||
def _system_partner() -> dict:
|
||||
return {"id": SYSTEM_PARTNER_ID, "name": SYSTEM_NAME, "avatar_url": None}
|
||||
|
||||
|
||||
# The welcome's opening line in every language — how a stored copy is recognised. It was rendered
|
||||
# once, in the language of the day it was created, so an account that later switched language kept
|
||||
# reading it in the old one (E31 S1: the owner's, in English). Recognised here, it is served in the
|
||||
# reader's language instead. Derived from WELCOME itself, so a change to an OPENING line, or a
|
||||
# language taken out of WELCOME, orphans the copies already stored: they are served as stored. The
|
||||
# one retired language (German, 2026-07) left none on prod (checked 2026-10-02).
|
||||
_WELCOME_OPENINGS = {text.partition("\n")[0] for text in WELCOME.values()}
|
||||
|
||||
|
||||
def _system_body(body: str | None, lang: str | None) -> str | None:
|
||||
"""A system message's text for a reader of `lang`: the welcome in their language, any other as
|
||||
stored."""
|
||||
if body is None or lang is None or body.partition("\n")[0] not in _WELCOME_OPENINGS:
|
||||
return body
|
||||
return WELCOME.get(lang, WELCOME["en"])
|
||||
|
||||
|
||||
def _serialize_msg(m: Message, lang: str | None = None) -> dict:
|
||||
"""`lang`: the reader's language, for a system message's text (a user message is ciphertext)."""
|
||||
return {
|
||||
"id": m.id,
|
||||
"kind": m.kind,
|
||||
"sender_id": m.sender_id,
|
||||
"recipient_id": m.recipient_id,
|
||||
"body": _system_body(m.body, lang) if m.kind == "system" else m.body,
|
||||
"ciphertext": m.ciphertext,
|
||||
"iv": m.iv,
|
||||
"read_at": iso(m.read_at),
|
||||
"created_at": iso(m.created_at),
|
||||
}
|
||||
|
||||
|
||||
def ensure_welcome(db: Session, user: User) -> None:
|
||||
"""Create the one-time system welcome message for a (human) user the first time it's needed.
|
||||
Idempotent: guarded on whether any system message already exists for them."""
|
||||
if user.is_demo:
|
||||
return
|
||||
exists = db.scalar(
|
||||
select(Message.id).where(Message.recipient_id == user.id, Message.kind == "system").limit(1)
|
||||
)
|
||||
if exists:
|
||||
return
|
||||
lang = (user.preferences or {}).get("language")
|
||||
body = WELCOME.get(lang, WELCOME["en"])
|
||||
m = Message(kind="system", sender_id=None, recipient_id=user.id, body=body)
|
||||
db.add(m)
|
||||
db.commit()
|
||||
db.refresh(m)
|
||||
manager.push(user.id, {"type": "message", "message": _serialize_msg(m, lang_for_user(user))})
|
||||
|
||||
|
||||
# --- E2EE key directory / blob store -----------------------------------------
|
||||
|
||||
|
||||
@router.get("/keys/me")
|
||||
def my_key(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
"""My own key bundle. `configured` says whether secure messaging is set up; if so the
|
||||
wrapped blob + params are returned so this device can unlock with the passphrase."""
|
||||
k = db.get(MessageKey, user.id)
|
||||
if k is None:
|
||||
return {"configured": False}
|
||||
return {
|
||||
"configured": True,
|
||||
"public_key": k.public_key,
|
||||
"wrapped_private_key": k.wrapped_private_key,
|
||||
"salt": k.salt,
|
||||
"wrap_iv": k.wrap_iv,
|
||||
"key_check": k.key_check,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/keys")
|
||||
def setup_keys(
|
||||
payload: KeySetupIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Store this user's public key + passphrase-wrapped private key. Set-once: changing the
|
||||
key would orphan every existing conversation key, so re-setup is rejected here."""
|
||||
if db.get(MessageKey, user.id) is not None:
|
||||
raise refused(409, "messaging_set_up")
|
||||
db.add(
|
||||
MessageKey(
|
||||
user_id=user.id,
|
||||
public_key=payload.public_key,
|
||||
wrapped_private_key=payload.wrapped_private_key,
|
||||
salt=payload.salt,
|
||||
wrap_iv=payload.wrap_iv,
|
||||
key_check=payload.key_check,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return {"ok": True, "configured": True}
|
||||
|
||||
|
||||
@router.post("/keys/reset")
|
||||
def reset_keys(
|
||||
payload: KeyResetIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Forgotten-passphrase escape hatch: drop this user's secure-messaging key so they can set a
|
||||
NEW passphrase (the set-once guard in setup_keys otherwise makes a lost passphrase a permanent
|
||||
wall). Destroys the user's entire E2EE history BY DESIGN — every conversation key is derived
|
||||
from this keypair, so replacing it orphans every stored ciphertext in both directions. Those
|
||||
now-undecryptable `user` messages are deleted here (a clean slate for both parties) rather than
|
||||
left as permanent "can't decrypt" clutter; `system` messages (plaintext) are untouched.
|
||||
|
||||
An account WITH a password must re-authenticate: a hijacked session must not be able to silently
|
||||
reset messaging and then read the victim's future messages under a new passphrase. A password-
|
||||
less (Google-only) account has nothing to re-verify, so it relies on its live session plus the
|
||||
client-side danger confirmation."""
|
||||
if not _reset_key_limiter.allow(str(user.id)):
|
||||
raise refused(429, "too_many_attempts")
|
||||
if user.password_hash and not verify_password(
|
||||
payload.current_password or "", user.password_hash
|
||||
):
|
||||
raise refused(403, "wrong_password")
|
||||
# Delete the now-dead ciphertext first (both directions), then the wrapped keypair itself so a
|
||||
# fresh setup is allowed. Order doesn't matter (no FK between them), but do it in one commit.
|
||||
db.execute(
|
||||
delete(Message).where(
|
||||
Message.kind == "user",
|
||||
or_(Message.sender_id == user.id, Message.recipient_id == user.id),
|
||||
)
|
||||
)
|
||||
key = db.get(MessageKey, user.id)
|
||||
if key is not None:
|
||||
db.delete(key)
|
||||
db.commit()
|
||||
return {"ok": True, "configured": False}
|
||||
|
||||
|
||||
@router.get("/keys/{user_id}")
|
||||
def public_key(
|
||||
user_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""A member's public key, for deriving the pairwise conversation key. MB2 (like get_thread):
|
||||
resolve the target only if they're messageable OR we already share a thread, else the same
|
||||
"User not found" as a non-existent id — so this can't be probed to enumerate user_ids or to
|
||||
fetch keys for suspended/deactivated/demo accounts."""
|
||||
cond = and_(
|
||||
Message.kind == "user",
|
||||
or_(
|
||||
and_(Message.sender_id == user.id, Message.recipient_id == user_id),
|
||||
and_(Message.sender_id == user_id, Message.recipient_id == user.id),
|
||||
),
|
||||
)
|
||||
p = db.get(User, user_id)
|
||||
has_history = db.scalar(select(Message.id).where(cond).limit(1))
|
||||
if p is None or (not is_messageable_user(p) and not has_history):
|
||||
raise not_found("user")
|
||||
k = db.get(MessageKey, user_id)
|
||||
if k is None:
|
||||
raise refused(404, "no_messaging")
|
||||
return {"user_id": user_id, "public_key": k.public_key}
|
||||
|
||||
|
||||
# --- directory + conversations -----------------------------------------------
|
||||
|
||||
|
||||
@router.get("/users")
|
||||
def list_recipients(
|
||||
user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> list[dict]:
|
||||
"""Member directory for the recipient picker: messageable members (except me) who have set
|
||||
up secure messaging (so a conversation key can be derived)."""
|
||||
rows = (
|
||||
db.execute(
|
||||
select(User)
|
||||
.join(MessageKey, MessageKey.user_id == User.id)
|
||||
.where(User.id != user.id, *messageable_clauses())
|
||||
.order_by(collated(func.coalesce(User.display_name, User.email), lang_for_user(user)))
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return [_serialize_user(u) for u in rows]
|
||||
|
||||
|
||||
@router.get("/conversations")
|
||||
def list_conversations(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
"""One entry per conversation: each user partner (E2EE) plus the system "Siftlode" thread,
|
||||
with the latest message + my unread count, newest first. Lazily creates the welcome on the
|
||||
first open. User-message previews are ciphertext — the client decrypts them."""
|
||||
ensure_welcome(db, user)
|
||||
# The conversation partner for a message, from this user's perspective (system → 0).
|
||||
partner_expr = case(
|
||||
(Message.kind == "system", SYSTEM_PARTNER_ID),
|
||||
(Message.sender_id == user.id, Message.recipient_id),
|
||||
else_=Message.sender_id,
|
||||
)
|
||||
mine = or_(Message.sender_id == user.id, Message.recipient_id == user.id)
|
||||
# Latest message per conversation in ONE query (DISTINCT ON, Postgres) instead of loading the
|
||||
# user's whole message history into Python — bounded by the number of conversations.
|
||||
latest = (
|
||||
db.execute(
|
||||
select(Message)
|
||||
.where(mine)
|
||||
.distinct(partner_expr)
|
||||
.order_by(partner_expr, Message.created_at.desc())
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
# Unread incoming, grouped per conversation.
|
||||
unread = dict(
|
||||
db.execute(
|
||||
select(partner_expr, func.count())
|
||||
.where(mine, Message.recipient_id == user.id, Message.read_at.is_(None))
|
||||
.group_by(partner_expr)
|
||||
).all()
|
||||
)
|
||||
|
||||
def _pid(m: Message) -> int:
|
||||
if m.kind == "system":
|
||||
return SYSTEM_PARTNER_ID
|
||||
return m.recipient_id if m.sender_id == user.id else m.sender_id
|
||||
|
||||
user_ids = [_pid(m) for m in latest if m.kind != "system"]
|
||||
partners = {}
|
||||
if user_ids:
|
||||
partners = {
|
||||
u.id: u for u in db.execute(select(User).where(User.id.in_(user_ids))).scalars().all()
|
||||
}
|
||||
items = []
|
||||
for m in latest:
|
||||
pid = _pid(m)
|
||||
if pid == SYSTEM_PARTNER_ID:
|
||||
partner = _system_partner()
|
||||
elif pid in partners:
|
||||
partner = _serialize_user(partners[pid])
|
||||
else:
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"partner": partner,
|
||||
"last_message": _serialize_msg(m, lang_for_user(user)),
|
||||
"unread": int(unread.get(pid, 0)),
|
||||
}
|
||||
)
|
||||
items.sort(key=lambda x: x["last_message"]["created_at"] or "", reverse=True)
|
||||
return {"items": items}
|
||||
|
||||
|
||||
@router.get("/conversations/{partner_id}")
|
||||
def get_thread(
|
||||
partner_id: int,
|
||||
mark_read: bool = True,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Full thread with a partner (or the system thread when partner_id == 0), oldest first.
|
||||
Stamps unread incoming messages read unless mark_read=false."""
|
||||
if partner_id == SYSTEM_PARTNER_ID:
|
||||
ensure_welcome(db, user)
|
||||
partner = _system_partner()
|
||||
cond = and_(Message.recipient_id == user.id, Message.kind == "system")
|
||||
else:
|
||||
p = db.get(User, partner_id)
|
||||
cond = and_(
|
||||
Message.kind == "user",
|
||||
or_(
|
||||
and_(Message.sender_id == user.id, Message.recipient_id == partner_id),
|
||||
and_(Message.sender_id == partner_id, Message.recipient_id == user.id),
|
||||
),
|
||||
)
|
||||
# MB2: don't leak arbitrary user identities. Resolve a partner only if they're messageable
|
||||
# OR we already share a thread (so a conversation with a since-suspended/deactivated user
|
||||
# still opens). Otherwise the same 404 as a non-existent id → no enumeration oracle.
|
||||
has_history = db.scalar(select(Message.id).where(cond).limit(1))
|
||||
if p is None or (not is_messageable_user(p) and not has_history):
|
||||
raise not_found("user")
|
||||
partner = _serialize_user(p)
|
||||
msgs = (
|
||||
db.execute(select(Message).where(cond).order_by(Message.created_at.asc())).scalars().all()
|
||||
)
|
||||
if mark_read:
|
||||
now = datetime.now(timezone.utc)
|
||||
changed = False
|
||||
for m in msgs:
|
||||
if m.recipient_id == user.id and m.read_at is None:
|
||||
m.read_at = now
|
||||
changed = True
|
||||
if changed:
|
||||
db.commit()
|
||||
# MB3: ping this user's OTHER tabs/devices that their unread dropped so the badge
|
||||
# refreshes live instead of waiting for the next 20s poll. The client re-fetches the
|
||||
# count on this signal, so no number is carried here.
|
||||
manager.push(user.id, {"type": "unread"})
|
||||
return {"partner": partner, "items": [_serialize_msg(m, lang_for_user(user)) for m in msgs]}
|
||||
|
||||
|
||||
@router.post("")
|
||||
def send_message(
|
||||
payload: SendIn,
|
||||
user: User = Depends(require_human),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Send an end-to-end-encrypted message. The server stores only the ciphertext the client
|
||||
produced, then pushes it live to the recipient (and the sender's other devices)."""
|
||||
if not _send_limiter.allow(f"msg:{user.id}"):
|
||||
raise refused(429, "message_rate")
|
||||
ct = (payload.ciphertext or "").strip()
|
||||
iv = (payload.iv or "").strip()
|
||||
if not ct or not iv:
|
||||
raise refused(400, "message_empty")
|
||||
if len(ct) > MAX_CIPHERTEXT:
|
||||
raise refused(400, "message_too_long")
|
||||
if payload.recipient_id == user.id:
|
||||
raise refused(400, "message_self")
|
||||
recipient = db.get(User, payload.recipient_id)
|
||||
if not is_messageable_user(recipient):
|
||||
raise not_found("recipient")
|
||||
if db.get(MessageKey, recipient.id) is None:
|
||||
raise refused(404, "no_messaging")
|
||||
m = Message(kind="user", sender_id=user.id, recipient_id=recipient.id, ciphertext=ct, iv=iv)
|
||||
db.add(m)
|
||||
db.commit()
|
||||
db.refresh(m)
|
||||
# Carry both parties so each recipient can open/flash the right dock window (the partner is
|
||||
# whichever of from/to isn't them).
|
||||
event = {
|
||||
"type": "message",
|
||||
"message": _serialize_msg(m),
|
||||
"from": _serialize_user(user),
|
||||
"to": _serialize_user(recipient),
|
||||
}
|
||||
manager.push(recipient.id, event)
|
||||
manager.push(user.id, event) # the sender's other open devices
|
||||
return _serialize_msg(m)
|
||||
|
||||
|
||||
@router.get("/unread_count")
|
||||
def unread_count(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
n = db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Message)
|
||||
.where(Message.recipient_id == user.id, Message.read_at.is_(None))
|
||||
)
|
||||
return {"count": n or 0}
|
||||
|
||||
|
||||
# --- live delivery -----------------------------------------------------------
|
||||
|
||||
|
||||
@router.websocket("/ws")
|
||||
async def messages_ws(ws: WebSocket) -> None:
|
||||
"""Live push channel: authenticated by the session cookie, registers the connection so
|
||||
sent messages reach this user's open tabs instantly. We don't consume client→server frames
|
||||
(sending goes through POST /api/messages); the receive loop only detects disconnect."""
|
||||
# Which signed-in account this socket acts as — the SHARED per-tab resolution (a wallet-gated
|
||||
# ?account=, the session default when none is named, nobody for an account no longer signed in
|
||||
# here). A browser WebSocket can't send the X-Siftlode-Account header, so resolved_user_id falls
|
||||
# through to the ?account= query param.
|
||||
uid, _ = resolved_user_id(ws)
|
||||
if not uid:
|
||||
await ws.close(code=1008)
|
||||
return
|
||||
db = SessionLocal()
|
||||
try:
|
||||
user = db.get(User, uid)
|
||||
# SA4: honour server-side session revocation on the live channel too — reject a cookie whose
|
||||
# recorded epoch is behind the account's current one (mirrors current_user). Without this a
|
||||
# copied cookie could keep receiving pushes after a password reset / "log out everywhere".
|
||||
epochs = ws.session.get("epochs") or {}
|
||||
epoch_ok = user is not None and int(epochs.get(str(uid), 0)) == (user.session_epoch or 0)
|
||||
ok = epoch_ok and is_messageable_user(user)
|
||||
finally:
|
||||
db.close()
|
||||
if not ok:
|
||||
await ws.close(code=1008)
|
||||
return
|
||||
await manager.connect(uid, ws)
|
||||
try:
|
||||
while True:
|
||||
await ws.receive_text()
|
||||
# silent-ok: the ordinary way a chat socket ends — a closed tab. The `finally` below is the
|
||||
# whole handling, and nothing failed.
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
finally:
|
||||
await manager.disconnect(uid, ws)
|
||||
@@ -1,9 +1,5 @@
|
||||
"""Per-user notification inbox (phase 1): the durable, server-backed center that coexists
|
||||
with the client-side transient bell. Read-only listing plus read/dismiss/clear mutations.
|
||||
|
||||
All endpoints use `current_user` (not `require_human`): notifications and their read/dismiss
|
||||
state are personal UI state with no quota or YouTube involvement, so the shared demo account
|
||||
manages its own inbox normally."""
|
||||
with the client-side transient bell. Read-only listing plus read/dismiss/clear mutations."""
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy import func, select, update
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""Plex route package — assembles the admin/browse/playback sub-routers.
|
||||
|
||||
Read endpoints (libraries/browse/show/image) are available to ANY authenticated user (demo +
|
||||
pure-Siftlode included) — this is an access layer to the Plex library WITHOUT requiring a plex.tv
|
||||
Read endpoints (libraries/browse/show/image) are available to ANY authenticated user
|
||||
(pure-Siftlode included) — this is an access layer to the Plex library WITHOUT requiring a plex.tv
|
||||
account, and playback is a local file. Admin endpoints test the connection and run the catalog sync.
|
||||
|
||||
Split out of the former single routes/plex.py (R12 S1). `router` keeps the same shape the app
|
||||
Split out of the former single routes/plex.py. `router` keeps the same shape the app
|
||||
wiring expects (`plex_routes.router`), so main.py is unchanged.
|
||||
"""
|
||||
|
||||
|
||||
@@ -285,7 +285,7 @@ def unified_library(
|
||||
episodes = []
|
||||
if tsq is not None and want_shows and field in ("all", "episode"):
|
||||
ep_st = aliased(PlexState)
|
||||
# In HU an episode is found by its Hungarian title too (E30: the overlay now fills it) — and
|
||||
# In HU an episode is found by its Hungarian title too (the overlay fills it) — and
|
||||
# still by its English one, like `vector_match` does for the grid.
|
||||
if field == "episode":
|
||||
ep_vec = func.to_tsvector(TS_CONFIG, func.coalesce(PlexItem.title, ""))
|
||||
@@ -360,8 +360,8 @@ def facets(
|
||||
|
||||
# The shows that pass the watch-state scope, asked ONCE. The per-user episode rollup behind it is a
|
||||
# full pass over the user's episodes (~10 ms on a prod-sized library), and each of the show groups
|
||||
# (genres, ratings, studios, years, rating max) asked it again: half of a filtered facets call
|
||||
# (E31 S4e). The same filter on the same rows, so the counts do not change
|
||||
# (genres, ratings, studios, years, rating max) asked it again: half of a filtered facets call.
|
||||
# The same filter on the same rows, so the counts do not change
|
||||
# (test_plex_facets_grid_parity).
|
||||
shows_in_scope: list[int] | None = None
|
||||
if show_ids and show in ("watched", "in_progress", "unwatched"):
|
||||
@@ -545,7 +545,7 @@ def facets(
|
||||
for c in split_csv(f.playable):
|
||||
if c in PLAYBACK_CLASSES:
|
||||
playable_counts.setdefault(c, 0)
|
||||
# Names in the reader's alphabet: the genres are Hungarian words for a Hungarian reader (E30 S1).
|
||||
# Names in the reader's alphabet: the genres are Hungarian words for a Hungarian reader.
|
||||
alpha = {w: i for i, w in enumerate(sorted_words(db, [*genre_counts, *studio_counts], lang))}
|
||||
return {
|
||||
"genres": [
|
||||
|
||||
@@ -153,7 +153,7 @@ def show_detail(
|
||||
all_cards.append(card)
|
||||
show_roll = rollup.rollup(all_cards)
|
||||
|
||||
# Cast + IMDb + tagline are served from the DB row (persisted by the cast backfill, E5 S5b). The one
|
||||
# Cast + IMDb + tagline are served from the DB row (persisted by the cast backfill). The one
|
||||
# remaining live call is `related` — a Plex recommendation hub we don't mirror — which also carries
|
||||
# the metadata used to lazily fill a not-yet-enriched show. Any live failure just drops `related` and
|
||||
# the lazy fill; the row-served detail below still renders (outage-resilient).
|
||||
@@ -213,7 +213,7 @@ def show_detail(
|
||||
"first": show_roll["first"],
|
||||
"episode_count": show_roll["episode_count"],
|
||||
"collection_keys": sh.collection_keys or [],
|
||||
# TMDB enrichment (E5 S2), served from the row.
|
||||
# TMDB enrichment, served from the row.
|
||||
"tmdb": plex_tmdb.serve_block(db, sh, lang=lang),
|
||||
},
|
||||
"seasons": [_season_block(se, by_season.get(se.id, [])) for se in seasons],
|
||||
@@ -353,7 +353,7 @@ _TRACK_TAIL = re.compile(r"\(([^()]+)\)\s*$")
|
||||
|
||||
|
||||
def _track_naming(s: dict) -> dict:
|
||||
"""What the player names a track from (E30). Plex's `displayTitle` is in the SERVER's language
|
||||
"""What the player names a track from. Plex's `displayTitle` is in the SERVER's language
|
||||
("English (AC3 5.1)", "magyar Forced", "中文 (EAC3 Stereo)"), so it is sent only as the fallback
|
||||
`label`; the client builds the name in the reader's language from the language tag, the
|
||||
forced / SDH flags and the codec tail. No title and no language → no label (the client numbers it)."""
|
||||
@@ -404,6 +404,10 @@ def item_detail(
|
||||
"type": m.get("type"),
|
||||
"start_s": round((m.get("startTimeOffset") or 0) / 1000, 1),
|
||||
"end_s": round((m.get("endTimeOffset") or 0) / 1000, 1),
|
||||
# An item can carry several credits blocks with scenes between them; Plex
|
||||
# flags the one that really ends it. Which credits may leave the item is the
|
||||
# player's rule (`lib/plexMarkers`), so the flag goes over as Plex sent it.
|
||||
"final": bool(m.get("final")),
|
||||
}
|
||||
)
|
||||
# Audio + subtitle tracks (ordinals among their stream type — used to -map the selection).
|
||||
@@ -538,6 +542,6 @@ def item_detail(
|
||||
"prev_id": prev_id,
|
||||
"next_id": next_id,
|
||||
"collections": _collection_strips(db, it, user.id, lang),
|
||||
# TMDB enrichment (E5 S2), served from the row — movies only (episodes aren't enriched).
|
||||
# TMDB enrichment, served from the row — movies only (episodes aren't enriched).
|
||||
"tmdb": plex_tmdb.serve_block(db, it, lang=lang) if it.kind == "movie" else None,
|
||||
}
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
"""Plex Discover: the rail-surface read-model (E5 S4).
|
||||
"""Plex Discover: the rail-surface read-model.
|
||||
|
||||
A consolidated payload for the dedicated Discover landing — a stack of horizontal rails:
|
||||
Continue watching · Recently added · Browse-by-genre tiles · Because you watched X — plus a
|
||||
separate, reshuffling Surprise-me sample. Serve-from-DB only: the recommendation rail reads the
|
||||
recs the S2 TMDB engine already persisted (`PlexItem.recommendations`), never a live TMDB call, and
|
||||
recs the TMDB engine already persisted (`PlexItem.recommendations`), never a live TMDB call, and
|
||||
every card comes from the same `movie_card`/`show_card` serializers the grid uses (so rails inherit
|
||||
S1's `art` + S3's rating/summary for free). The rails carry EN metadata (like the other non-grid
|
||||
their `art` + rating/summary for free). The rails carry EN metadata (like the other non-grid
|
||||
card paths); the HU overlay coalesces only in the main grid.
|
||||
|
||||
The sidebar metadata filters (genre / rating / year / collection / …) narrow EVERY rail too, so the
|
||||
@@ -114,8 +114,8 @@ def _recently_added(
|
||||
else []
|
||||
)
|
||||
# Merge the two type-homogeneous lists by added_at, then serialize each side in one batch so the
|
||||
# per-card state lookups stay batched (no N+1). Same `or NO_TS` key as the two rails below — the
|
||||
# E24 review found this one still on the older `(is not None, value)` form, i.e. one rule in two
|
||||
# per-card state lookups stay batched (no N+1). Same `or NO_TS` key as the two rails below — a
|
||||
# review found this one still on the older `(is not None, value)` form, i.e. one rule in two
|
||||
# spellings, with the two comments explaining it differently.
|
||||
tagged = [(m.added_at, "movie", m) for m in movies] + [(s.added_at, "show", s) for s in shows]
|
||||
tagged.sort(key=lambda t: t[0] or rollup.NO_TS, reverse=True)
|
||||
|
||||
@@ -176,10 +176,10 @@ def person_detail(
|
||||
people who share a name share a page. Each section is capped at _PERSON_SECTION_MAX. An unknown name
|
||||
simply returns empty sections.
|
||||
|
||||
E5 S6: the first time a page is opened, the person's TMDB bio + "Known for" credits are lazily
|
||||
fetched (from the S2-linked `tmdb_person_id`) and persisted, then served from the row on every open.
|
||||
The first time a page is opened, the person's TMDB bio + "Known for" credits are lazily
|
||||
fetched (from the linked `tmdb_person_id`) and persisted, then served from the row on every open.
|
||||
The fetch runs OFF the request path (a background task); the first open serves whatever's in the row
|
||||
with `enriching` set, and the page asks again until the bio lands. E30 S6: the people_enrich job
|
||||
with `enriching` set, and the page asks again until the bio lands. The people_enrich job
|
||||
fetches everyone ahead of time, so this lazy path is only for a person the job hasn't reached yet.
|
||||
"""
|
||||
lang = norm_lang(lang)
|
||||
@@ -187,7 +187,7 @@ def person_detail(
|
||||
# Lazy one-time bio enrich, scheduled AFTER the response so the TMDB call is never on the serve path.
|
||||
# Gated by a per-person cooldown so concurrent/rapid opens (or repeated opens while TMDB is degraded)
|
||||
# can't pile background tasks onto the shared threadpool.
|
||||
# `enriching` tells the page the bio is on its way (E30 S6): it asks again shortly instead of showing
|
||||
# `enriching` tells the page the bio is on its way: it asks again shortly instead of showing
|
||||
# an empty page until the next open. True while a fetch is pending, including one an earlier open
|
||||
# already reserved.
|
||||
enriching = (
|
||||
|
||||
@@ -15,7 +15,7 @@ from fastapi import (
|
||||
Response,
|
||||
)
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -23,7 +23,6 @@ from sqlalchemy.orm import Session
|
||||
from app.auth import current_user, resolved_user_id
|
||||
from app.db import SessionLocal, get_db
|
||||
from app.models import (
|
||||
POSITION_MAX_S,
|
||||
PlexItem,
|
||||
PlexSeason,
|
||||
PlexShow,
|
||||
@@ -39,7 +38,7 @@ from app.plex import watch_sync as plex_watch
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.routes.plex._shared import item_or_404
|
||||
from app.refusal import refused
|
||||
from app.routes._common import not_found
|
||||
from app.routes._common import ProgressIn, not_found
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
|
||||
@@ -160,16 +159,16 @@ def subtitle(
|
||||
return Response(subtitles.shift_vtt(hit, offset), media_type="text/vtt")
|
||||
|
||||
|
||||
# Resume/watched thresholds. _PROGRESS_MIN_S matches the feed's PROGRESS_MIN_SECONDS (5), but
|
||||
# _FINISH_MARGIN_S is DELIBERATELY larger than the feed's FINISH_MARGIN_SECONDS (30 vs 10): this is
|
||||
# long-form content (movies/episodes with credits) and crossing it MARKS THE ITEM WATCHED, whereas
|
||||
# the feed's margin only clears a short video's resume bar — different domains, intentional divergence,
|
||||
# not drift. Keep them separate on purpose (see routes/feed.py).
|
||||
# Resume/watched thresholds. _PROGRESS_MIN_S matches PROGRESS_MIN_SECONDS (5), but _FINISH_MARGIN_S is
|
||||
# DELIBERATELY larger than FINISH_MARGIN_SECONDS (30 vs 10): this is long-form content
|
||||
# (movies/episodes with credits) and crossing it MARKS THE ITEM WATCHED, whereas the feed's and the
|
||||
# library's margin only clears a resume bar — different domains, intentional divergence, not drift.
|
||||
# Keep them separate on purpose (see routes/_common.py).
|
||||
_PROGRESS_MIN_S = 5
|
||||
|
||||
_FINISH_MARGIN_S = 30
|
||||
|
||||
# S2 write-back throttle: the resume position is now pushed to Plex on the periodic (non-final)
|
||||
# Write-back throttle: the resume position is now pushed to Plex on the periodic (non-final)
|
||||
# checkpoint too — not only on the final pause/unmount flush — so Plex (the source of truth) stays
|
||||
# current in near-real-time. To keep a single watch from spraying `/:/timeline` calls, a push is
|
||||
# rate-limited to at most one per item per user every _PUSH_THROTTLE_S seconds; a `final` checkpoint
|
||||
@@ -337,16 +336,9 @@ def _mark_finished(db: Session, user_id: int, item_id: int, now: datetime) -> bo
|
||||
return False
|
||||
|
||||
|
||||
class ItemProgressIn(BaseModel):
|
||||
# Finite only: JSON parsing here accepts `Infinity`/`NaN`, and the handler's `int()` turned
|
||||
# one into a 500.
|
||||
model_config = ConfigDict(allow_inf_nan=False)
|
||||
class ItemProgressIn(ProgressIn):
|
||||
"""The shared checkpoint (its bounds included), plus whether the player is leaving the item."""
|
||||
|
||||
# float, truncated in the handler — the player may post a raw currentTime (a float); a plain
|
||||
# `int` field would 422 it on the periodic progress checkpoint (the old `int(...)` truncated).
|
||||
# Bounded above by the int4 column (see POSITION_MAX_S): past it the flush was a 500.
|
||||
position_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
duration_seconds: float = Field(0, le=POSITION_MAX_S)
|
||||
final: bool = False
|
||||
|
||||
|
||||
@@ -360,7 +352,7 @@ def item_progress(
|
||||
) -> dict:
|
||||
"""Checkpoint the resume position while playing (near the end → mark watched + clear it).
|
||||
|
||||
S2 write-back: the resume position is mirrored to Plex (the source of truth) on the periodic
|
||||
Write-back: the resume position is mirrored to Plex (the source of truth) on the periodic
|
||||
checkpoint too, not only on the final flush — throttled to ≥_PUSH_THROTTLE_S per item (see
|
||||
`_should_push_resume`). `final` (sent on pause / seek / pagehide / unmount) pushes immediately and
|
||||
resets the throttle window, so Plex sees a settled position the instant the user pauses or seeks."""
|
||||
@@ -637,12 +629,27 @@ def stream_session(
|
||||
# The session id is part of the path (not a query param) so the playlist's RELATIVE segment
|
||||
# names resolve back into the same session — see `stream_hls_file` for why it has to.
|
||||
"url": f"/api/plex/stream/{it.rating_key}/hls/{s.id}/{s.entry}",
|
||||
# What the player names when it no longer wants the session (POST …/session/{id}/stop).
|
||||
"id": s.id,
|
||||
"transcode": s.transcode,
|
||||
"hw": s.hw,
|
||||
"tonemap": s.tonemap,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/stream/{rating_key}/session/{session_id}/stop")
|
||||
def stream_session_stop(
|
||||
rating_key: str,
|
||||
session_id: str,
|
||||
user: User = Depends(current_user),
|
||||
) -> dict:
|
||||
"""The player no longer wants this session (it left the item, closed, or the page unloaded):
|
||||
its ffmpeg stops now instead of at the idle reaper. A POST, so the player can send it as a
|
||||
keepalive beacon from a page being unloaded. Stops nothing but the caller's own session, and
|
||||
only while it is still the one registered for the item (see `stop_session`)."""
|
||||
return {"ok": True, "stopped": plex_stream.stop_session(user.id, rating_key, session_id)}
|
||||
|
||||
|
||||
@router.get("/stream/{rating_key}/file")
|
||||
def stream_file(
|
||||
rating_key: str,
|
||||
@@ -696,7 +703,7 @@ def stream_hls_file(
|
||||
(viewer, rating_key) and a seek-restart replaces the viewer's registered one, while the DIRECTORY
|
||||
is per (key, viewer, offset, tag) — so resolving by the key alone handed the old player the new
|
||||
session's `seg_12.ts`, produced from a different offset under `-copyts`: wrong
|
||||
bytes with a 200, which the player has no way to notice (deep-review round 5). A request for a
|
||||
bytes with a 200, which the player has no way to notice. A request for a
|
||||
session that is no longer the registered one is a 404, and the player reclaims at the playhead.
|
||||
|
||||
That reclaim is also why a request with no live session is no longer auto-started here: the
|
||||
@@ -728,7 +735,7 @@ def stream_hls_file(
|
||||
# `proc` so a session whose ffmpeg has already exited answers at once instead of polling out the
|
||||
# whole timeout: this is a SYNC route on the ~40-slot threadpool, holding a pooled DB connection
|
||||
# with it, so every such wait is a worker (and a connection) parked for 20 s — an ordinary seek
|
||||
# past the produced region was enough to do it (round 5).
|
||||
# past the produced region was enough to do it.
|
||||
if not plex_stream.wait_for(path, timeout, proc=s.proc):
|
||||
raise refused(404, "not_ready")
|
||||
try:
|
||||
|
||||
@@ -296,7 +296,7 @@ def _items_by_rks(db: Session, rks: list) -> list[PlexItem]:
|
||||
|
||||
|
||||
class PlexItemAddBulkIn(PlexItemKeysIn):
|
||||
# Where each key goes back (E28 S3e2, the list page's "Undo" of a removal): an index over the
|
||||
# Where each key goes back (the list page's "Undo" of a removal): an index over the
|
||||
# ORDERED rows, one per key. The two-call shape (append, then send the whole order) leaves the rows
|
||||
# at the end whenever the second call fails, and the order it sends would overwrite anything
|
||||
# another tab changed in between.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
"""Public, login-free surface for shared download links (the `/watch/{token}` player page).
|
||||
|
||||
These routes are deliberately OUTSIDE the auth/`require_human` gate — the unguessable token IS the
|
||||
These routes are deliberately OUTSIDE the auth gate — the unguessable token IS the
|
||||
credential (a capability URL). They only ever expose one file's stream + minimal metadata, never
|
||||
the app or any account data. Password-protected links exchange the password once at `/unlock` for
|
||||
a short-lived signed grant that the media URL carries (a `<video src>` can't send a header).
|
||||
@@ -16,6 +16,7 @@ from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, DownloadLink, MediaAsset
|
||||
from app.ratelimit import RateLimiter, client_key
|
||||
from app.security import verify_password
|
||||
@@ -46,6 +47,17 @@ def _link_or_404(db: Session, token: str) -> DownloadLink:
|
||||
def _ready_target(db: Session, link: DownloadLink) -> tuple[DownloadJob, MediaAsset]:
|
||||
job = db.get(DownloadJob, link.job_id)
|
||||
asset = db.get(MediaAsset, job.asset_id) if job and job.asset_id else None
|
||||
if (
|
||||
job is not None
|
||||
and asset is not None
|
||||
and asset.status in ("pending", "downloading")
|
||||
and job.status in (JobStatus.QUEUED, JobStatus.RUNNING)
|
||||
):
|
||||
# Being (re)made — a re-download, or a conversion — so the link works again once it is ready:
|
||||
# its own reason, so the page can say "try again shortly" rather than "this link is gone".
|
||||
# Only while the job is still going: a cancelled or paused rebuild leaves the asset pending
|
||||
# with nothing working on it, and "try again in a few minutes" would never come true.
|
||||
raise refused(404, "video_preparing")
|
||||
if job is None or asset is None or asset.status != "ready" or not asset.rel_path:
|
||||
raise refused(404, "video_gone")
|
||||
# Verify the file is actually on disk here (not just flagged ready), so the watch page never
|
||||
|
||||
@@ -4,15 +4,14 @@ from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import require_human
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import SavedView, User
|
||||
from app.routes._common import owned_or_404
|
||||
from app.refusal import refused
|
||||
|
||||
# Saved "smart views": per-user named snapshots of a module's filters (feed or plex, keyed by
|
||||
# `source`). Gated behind require_human so the shared demo account can't pollute a communal list
|
||||
# (mirrors how other personal-state writes are scoped).
|
||||
# `source`).
|
||||
router = APIRouter(prefix="/api/saved-views", tags=["saved-views"])
|
||||
|
||||
# The modules that own saved views. A view is scoped to exactly one; the default is "feed" so a caller
|
||||
@@ -37,7 +36,7 @@ def _serialize(v: SavedView) -> dict:
|
||||
@router.get("")
|
||||
def list_views(
|
||||
source: str = "feed",
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> list[dict]:
|
||||
source = _norm_source(source)
|
||||
@@ -55,7 +54,7 @@ def list_views(
|
||||
|
||||
# A feed view's tag pick is read by more than the feed: the autotag job keeps every tag a view names
|
||||
# (sync/autotag.py). Held here to the shape the feed sends — a list of tag ids, at most this many — so
|
||||
# a hand-made body (`"tags": 5`, or 60k ids) can't break that job for every user (E32 review).
|
||||
# a hand-made body (`"tags": 5`, or 60k ids) can't break that job for every user.
|
||||
_MAX_VIEW_TAGS = 500
|
||||
|
||||
|
||||
@@ -81,7 +80,7 @@ class CreateViewIn(BaseModel):
|
||||
@router.post("")
|
||||
def create_view(
|
||||
body: CreateViewIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
name = body.name.strip()
|
||||
@@ -124,7 +123,7 @@ class UpdateViewIn(BaseModel):
|
||||
def update_view(
|
||||
view_id: int,
|
||||
body: UpdateViewIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
view = owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
@@ -163,7 +162,7 @@ def update_view(
|
||||
@router.delete("/{view_id}")
|
||||
def delete_view(
|
||||
view_id: int,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
view = owned_or_404(db, SavedView, view_id, user, what="view")
|
||||
@@ -179,7 +178,7 @@ class ReorderViewsIn(BaseModel):
|
||||
@router.post("/reorder")
|
||||
def reorder_views(
|
||||
body: ReorderViewsIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
ids = body.ids
|
||||
|
||||
@@ -5,8 +5,8 @@ with the normal feed cards + in-app player and gain per-user state (watch / save
|
||||
source is admin-selectable (``search_source``): **scrape** (InnerTube, the website's own
|
||||
endpoint, zero API quota — the default) or **api** (search.list, **100 quota units per page**).
|
||||
Either way it's gated behind a per-user daily cap, the API source additionally behind a budget
|
||||
pre-check, and only real users (never the shared demo account) may call it. The result ids are
|
||||
enriched through the cheap shared videos.list path (1 unit / 50) regardless of source.
|
||||
pre-check. The result ids are enriched through the cheap shared videos.list path (1 unit / 50)
|
||||
regardless of source.
|
||||
|
||||
Ingested results are flagged (videos.via_search / channels.from_search) so the Library can hide
|
||||
this search-discovered "noise" by default. Shorts and currently-live/upcoming videos are never
|
||||
@@ -23,7 +23,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app import logs, quota, sysconfig
|
||||
from app.auth import require_human
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
@@ -251,7 +251,7 @@ def search_youtube(
|
||||
q: str | None = None,
|
||||
cursor: str | None = None,
|
||||
limit: int = 20,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Live YouTube search results materialised into the catalog. `limit` is how many results to
|
||||
@@ -402,7 +402,7 @@ class ClearSearchIn(BaseModel):
|
||||
@router.post("/clear")
|
||||
def clear_search(
|
||||
body: ClearSearchIn,
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Discard the given live-search results "as if never added": drop this user's search-finds
|
||||
|
||||
@@ -71,10 +71,10 @@ def setup_admin(body: SetupAdminIn, db: Session = Depends(require_setup)) -> dic
|
||||
raise refused(400, "invalid_email")
|
||||
if len(password) < PASSWORD_MIN_LEN:
|
||||
raise refused(400, "weak_password", min=PASSWORD_MIN_LEN)
|
||||
# Resolve canonically, not by the literal string (DX-17). The wizard can be re-run, and E9.5's
|
||||
# whole point is that two spellings of one Gmail mailbox are one person — matching literally here
|
||||
# would let a second run under the other spelling create the DUPLICATE admin row the epic exists
|
||||
# to prevent. Every other lookup went canonical in S3; this one was outside that diff.
|
||||
# Resolve canonically, not by the literal string (DX-17). The wizard can be re-run, and the
|
||||
# whole point of canonical email is that two spellings of one Gmail mailbox are one person —
|
||||
# matching literally here would let a second run under the other spelling create the DUPLICATE
|
||||
# admin row it exists to prevent. Every other lookup was canonical already; this one was missed.
|
||||
user = db.execute(
|
||||
select(User).where(User.email_canonical == canonical_email(email))
|
||||
).scalar_one_or_none()
|
||||
|
||||
@@ -4,7 +4,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, quota, state
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user, current_user, has_read_scope, require_human
|
||||
from app.auth import admin_user, current_user, has_read_scope
|
||||
from app.db import get_db
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
from app.routes._common import youtube_refusal
|
||||
@@ -26,7 +26,7 @@ _sync_limiter = RateLimiter(max_events=3, window_seconds=600)
|
||||
|
||||
|
||||
@router.post("/subscriptions")
|
||||
def sync_subscriptions(user: User = Depends(require_human), db: Session = Depends(get_db)) -> dict:
|
||||
def sync_subscriptions(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
if not has_read_scope(user):
|
||||
# No YouTube read grant yet — the onboarding wizard hasn't been completed.
|
||||
raise refused(403, "youtube_read_scope")
|
||||
@@ -37,7 +37,7 @@ def sync_subscriptions(user: User = Depends(require_human), db: Session = Depend
|
||||
result = import_subscriptions(db, user)
|
||||
except YouTubeError as exc:
|
||||
# The onboarding wizard's first call: an unguarded failure here was a raw 500 and the
|
||||
# blocking "Server error" dialog, with nothing the user could act on (E9 F1).
|
||||
# blocking "Server error" dialog, with nothing the user could act on.
|
||||
db.rollback()
|
||||
raise youtube_refusal(exc) from exc
|
||||
result["quota_used_estimate"] = m.used
|
||||
@@ -138,7 +138,7 @@ def my_status(user: User = Depends(current_user), db: Session = Depends(get_db))
|
||||
|
||||
@router.post("/deep-all")
|
||||
def deep_all(
|
||||
user: User = Depends(require_human),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
on: bool = True,
|
||||
) -> dict:
|
||||
|
||||
+16
-32
@@ -13,9 +13,8 @@ from app import audit, debugsession, logs, logsink, progress, quota
|
||||
from app.audit import AuditAction
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.demo import reset_demo_state
|
||||
from app.downloads.gc import run_download_gc
|
||||
from app.models import SchedulerSetting, User
|
||||
from app.models import SchedulerSetting
|
||||
from app.plex.sync import sync as run_plex_sync
|
||||
from app.plex.tmdb import run_enrich as run_tmdb_enrich
|
||||
from app.plex.tmdb import run_people_enrich
|
||||
@@ -63,7 +62,6 @@ JOB_INTERVALS: dict[str, int] = {
|
||||
"subscriptions": settings.subscriptions_resync_minutes,
|
||||
"playlist_sync": settings.playlist_sync_minutes,
|
||||
"maintenance": settings.maintenance_interval_minutes,
|
||||
"demo_reset": settings.demo_reset_minutes,
|
||||
"explore_cleanup": settings.explore_cleanup_minutes,
|
||||
"download_gc": settings.download_gc_minutes,
|
||||
"plex_sync": settings.plex_sync_interval_min,
|
||||
@@ -108,10 +106,6 @@ def apply_interval(db, job_id: str, minutes: int) -> int:
|
||||
return minutes
|
||||
|
||||
|
||||
def _now_iso() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _record(name: str, **fields) -> None:
|
||||
with _activity_lock:
|
||||
_activity.setdefault(name, {}).update(fields)
|
||||
@@ -139,8 +133,7 @@ def _notify_done(
|
||||
def _run_changed(result) -> bool:
|
||||
"""Did an automatic job run actually do something worth an audit row? (No-op polls don't.)
|
||||
A "change" = a truthy NUMERIC count; status-string dicts like {"skipped": "disabled"} (Plex
|
||||
off — the default) or {"skipped": "no demo account"} are no-ops, not changes, so they don't
|
||||
flood the trail every interval."""
|
||||
off — the default) are no-ops, not changes, so they don't flood the trail every interval."""
|
||||
if result is None:
|
||||
return False
|
||||
if isinstance(result, dict):
|
||||
@@ -161,7 +154,7 @@ def _audit_run(db, actor_id, job_id, status, summary, result) -> None:
|
||||
they changed something, so routine no-op polls don't spam the trail. Best-effort."""
|
||||
if actor_id is None and status == "ok" and not _run_changed(result):
|
||||
return
|
||||
# The result as data too (E30): the audit log names it in the reader's language, as the inbox
|
||||
# The result as data too: the audit log names it in the reader's language, as the inbox
|
||||
# does; `summary` stays the English line of the row's own summary column.
|
||||
data = _result_data(result)
|
||||
try:
|
||||
@@ -190,7 +183,7 @@ def _job(name: str, fn) -> None:
|
||||
db = SessionLocal()
|
||||
actor_id = _manual_actor.get() # set only for a manual "run now" trigger
|
||||
_record(
|
||||
name, last_started=_now_iso(), last_error=None, progress=None
|
||||
name, last_started=datetime.now(timezone.utc).isoformat(), last_error=None, progress=None
|
||||
) # running already claimed
|
||||
|
||||
def sink(current, total, phase):
|
||||
@@ -202,7 +195,7 @@ def _job(name: str, fn) -> None:
|
||||
_record(
|
||||
name,
|
||||
status="skipped",
|
||||
last_finished=_now_iso(),
|
||||
last_finished=datetime.now(timezone.utc).isoformat(),
|
||||
last_result="paused",
|
||||
result=_PAUSED,
|
||||
progress=None,
|
||||
@@ -219,7 +212,7 @@ def _job(name: str, fn) -> None:
|
||||
_record(
|
||||
name,
|
||||
status="ok",
|
||||
last_finished=_now_iso(),
|
||||
last_finished=datetime.now(timezone.utc).isoformat(),
|
||||
last_result=summary,
|
||||
result=data,
|
||||
progress=None,
|
||||
@@ -239,7 +232,7 @@ def _job(name: str, fn) -> None:
|
||||
_record(
|
||||
name,
|
||||
status="error",
|
||||
last_finished=_now_iso(),
|
||||
last_finished=datetime.now(timezone.utc).isoformat(),
|
||||
last_error=err,
|
||||
result=failure,
|
||||
progress=None,
|
||||
@@ -259,7 +252,7 @@ _PAUSED = {"skipped": "sync paused"}
|
||||
|
||||
|
||||
def _result_data(result) -> dict | None:
|
||||
"""A job's result as the client names it in the reader's language (E30): the dict itself (counts,
|
||||
"""A job's result as the client names it in the reader's language: the dict itself (counts,
|
||||
a `skipped` / `reason` word, a nested dict per phase), `enrich`'s bare count as one. The one-line
|
||||
`_summarize` is the English fallback."""
|
||||
if isinstance(result, dict):
|
||||
@@ -370,17 +363,6 @@ def _maintenance_job() -> None:
|
||||
_job("maintenance", work)
|
||||
|
||||
|
||||
def _demo_reset_job() -> None:
|
||||
# Auto-clean the shared demo sandbox. No-op (and never creates one) if there's no demo account.
|
||||
def work(db):
|
||||
demo = db.query(User).filter(User.is_demo.is_(True)).one_or_none()
|
||||
if demo is None:
|
||||
return {"skipped": "no demo account"}
|
||||
return {"playlists_seeded": reset_demo_state(db, demo)}
|
||||
|
||||
_job("demo_reset", work)
|
||||
|
||||
|
||||
def _explore_cleanup_job() -> None:
|
||||
# Reclaim un-kept ephemeral discovery content (explored channels + live-search results) after
|
||||
# their grace periods. 0 quota (pure DB), so no quota attribution.
|
||||
@@ -419,18 +401,21 @@ def _tmdb_enrich_job() -> None:
|
||||
|
||||
|
||||
def _people_enrich_job() -> None:
|
||||
# Make the people local (E30 S6): a batch of person bios from TMDB and a batch of portraits resized
|
||||
# Make the people local: a batch of person bios from TMDB and a batch of portraits resized
|
||||
# through Plex, so a person page and a cast strip never wait on an outside service.
|
||||
_job("people_enrich", run_people_enrich)
|
||||
|
||||
|
||||
def _audit_gc_job() -> None:
|
||||
# Prune audit-log rows older than the admin-set retention (audit_retention_days; 0 = keep
|
||||
# forever). Pure DB, no quota.
|
||||
# The history cleanup: audit-log rows past audit_retention_days and YouTube quota history past
|
||||
# quota_retention_days (0 = keep forever, each). Pure DB, no quota.
|
||||
def work(db):
|
||||
from app import sysconfig
|
||||
|
||||
return {"reaped": audit.gc(db, sysconfig.get_int(db, "audit_retention_days"))}
|
||||
reaped = audit.gc(db, sysconfig.get_int(db, "audit_retention_days"))
|
||||
quota_reaped = quota.gc(db, sysconfig.get_int(db, "quota_retention_days"))
|
||||
db.commit()
|
||||
return {"reaped": reaped, "quota_reaped": quota_reaped}
|
||||
|
||||
_job("audit_gc", work)
|
||||
|
||||
@@ -443,7 +428,7 @@ def _log_gc_job() -> None:
|
||||
|
||||
days = sysconfig.get_int(db, "log_retention_days")
|
||||
reaped = logsink.gc(db, days)
|
||||
# Debug sessions (E16) go with the rows they bundle; an open one past its cap is closed.
|
||||
# Debug sessions go with the rows they bundle; an open one past its cap is closed.
|
||||
closed = debugsession.sweep(db)
|
||||
sessions = debugsession.gc(db, days)
|
||||
db.commit()
|
||||
@@ -463,7 +448,6 @@ JOB_FUNCS: dict[str, Callable[[], None]] = {
|
||||
"subscriptions": _subscriptions_job,
|
||||
"playlist_sync": _playlist_sync_job,
|
||||
"maintenance": _maintenance_job,
|
||||
"demo_reset": _demo_reset_job,
|
||||
"explore_cleanup": _explore_cleanup_job,
|
||||
"download_gc": _download_gc_job,
|
||||
"plex_sync": _plex_sync_job,
|
||||
|
||||
@@ -116,7 +116,7 @@ _TOPIC_SLUGS = {
|
||||
"Fashion": "Fashion & Beauty",
|
||||
"Physical_attractiveness": "Fashion & Beauty",
|
||||
# YouTube links the British page (…/wiki/Humour); the key was "Humor", so no channel ever got Comedy
|
||||
# from its topic (E30).
|
||||
# from its topic.
|
||||
"Humour": "Comedy",
|
||||
"Entertainment": "Entertainment",
|
||||
"Film": "Film",
|
||||
@@ -300,11 +300,11 @@ def _cleanup_orphan_system_tags(db: Session) -> int:
|
||||
"""Delete system tags that no longer link to any channel (e.g. after a mapping change) — but not
|
||||
one a saved feed view filters on. Views hold tag IDS, and a deleted tag comes back under a new id
|
||||
when a channel earns it again, so the view's filter would point at nothing; the feed drops an
|
||||
unknown id, and the view showed the whole feed under its old name (E32 S2).
|
||||
unknown id, and the view showed the whole feed under its old name.
|
||||
|
||||
The views are read defensively, and the kept ids are filtered here rather than in SQL: a view
|
||||
stored before its route checked the shape (or written around it) must not raise for everyone's
|
||||
job, nor push a `NOT IN` past the driver's bind-parameter limit (E32 review)."""
|
||||
job, nor push a `NOT IN` past the driver's bind-parameter limit."""
|
||||
in_views = {
|
||||
t
|
||||
for (filters,) in db.execute(select(SavedView.filters).where(SavedView.source == "feed"))
|
||||
|
||||
@@ -8,7 +8,7 @@ stay private to users who explored (or subscribe to) the channel — the feed qu
|
||||
otherwise the cleanup job hard-deletes the channel and its untouched ephemeral videos after a
|
||||
grace period (mirrors the maintenance job's grace-then-delete safety)."""
|
||||
|
||||
from datetime import timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -24,7 +24,6 @@ from app.models import (
|
||||
VideoState,
|
||||
)
|
||||
from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
|
||||
from app.utils import now_utc as _now
|
||||
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
@@ -59,7 +58,7 @@ def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
|
||||
if not videos:
|
||||
return 0
|
||||
items = {it["id"]: it for it in yt.get_videos([v.id for v in videos])}
|
||||
now = _now()
|
||||
now = datetime.now(timezone.utc)
|
||||
for video in videos:
|
||||
item = items.get(video.id)
|
||||
if item is None:
|
||||
@@ -115,7 +114,7 @@ def purge_unkept_explored(db: Session) -> dict:
|
||||
or surfaced by the user's own search (SearchFind) — these survive even if the channel goes.
|
||||
A channel row is dropped only once it holds no videos at all. 0 quota."""
|
||||
grace_days = sysconfig.get_int(db, "explore_grace_days")
|
||||
cutoff = _now() - timedelta(days=grace_days)
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=grace_days)
|
||||
|
||||
expired = db.execute(
|
||||
delete(ExploredChannel).where(ExploredChannel.created_at < cutoff)
|
||||
@@ -166,7 +165,7 @@ def purge_unkept_search(db: Session, grace_days: int | None = None) -> dict:
|
||||
SearchFinds. `grace_days=0` purges immediately (the admin "purge now" path); None uses config.
|
||||
0 quota."""
|
||||
grace = sysconfig.get_int(db, "search_grace_days") if grace_days is None else grace_days
|
||||
cutoff = _now() - timedelta(days=grace)
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=grace)
|
||||
state_exists = select(VideoState.id).where(VideoState.video_id == Video.id).exists()
|
||||
pl_exists = select(PlaylistItem.id).where(PlaylistItem.video_id == Video.id).exists()
|
||||
sub_exists = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
|
||||
|
||||
@@ -20,7 +20,7 @@ Quota: phase 1 costs ~(flagged / 50) units, phase 2 ~(batch / 50). Both respect
|
||||
backfill reserve as the other jobs so interactive syncs never starve.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -31,7 +31,6 @@ from app.models import Playlist, PlaylistItem, Video
|
||||
from app.notifications import create_notification
|
||||
from app.sync.runner import get_service_user
|
||||
from app.sync.videos import apply_video_details, parse_dt
|
||||
from app.utils import now_utc as _now
|
||||
from app.youtube.client import YouTubeClient
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
@@ -60,7 +59,7 @@ def _is_abandoned_upcoming(item: dict) -> bool:
|
||||
scheduled = parse_dt(live.get("scheduledStartTime"))
|
||||
if scheduled is None:
|
||||
return False
|
||||
return _now() - scheduled > timedelta(days=2)
|
||||
return datetime.now(timezone.utc) - scheduled > timedelta(days=2)
|
||||
|
||||
|
||||
def _evaluate(video: Video, item: dict | None) -> str | None:
|
||||
@@ -191,7 +190,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient, now: datetime) -> dict:
|
||||
.where(Video.unavailable_since.is_(None))
|
||||
# Not one this run already checked: with fewer available videos than the batch, the
|
||||
# loop used to wrap around and re-fetch the same ones until `checked` reached the
|
||||
# batch, spending quota on answers it already had (E32 S2).
|
||||
# batch, spending quota on answers it already had.
|
||||
.where(or_(Video.last_checked_at.is_(None), Video.last_checked_at < now))
|
||||
# NULLS FIRST = never-checked first, then least-recently-checked; equivalent to the old
|
||||
# `(last_checked_at IS NULL) DESC, last_checked_at ASC` but in the exact shape of the
|
||||
@@ -225,7 +224,7 @@ def run_maintenance(db: Session) -> dict:
|
||||
return {"reason": "no service user"}
|
||||
# One `now` for the whole run: each phase taking its own let a video phase 1 had just recovered
|
||||
# (stamped with phase 1's) pass phase 2's "not checked this run" test and be fetched twice.
|
||||
now = _now()
|
||||
now = datetime.now(timezone.utc)
|
||||
with YouTubeClient(db, user) as yt:
|
||||
phase1 = _recheck_flagged(db, yt, now)
|
||||
phase2 = (
|
||||
|
||||
@@ -168,7 +168,7 @@ def _load_baseline(pl: Playlist) -> tuple[list[str] | None, set[str]]:
|
||||
|
||||
def parse_baseline(pl: Playlist) -> list[str] | None:
|
||||
"""The last-synced ordered id set, or None when no baseline has been recorded yet (a fresh
|
||||
export, or a link created before S7). None makes the first merge union losslessly."""
|
||||
export, or an older link). None makes the first merge union losslessly."""
|
||||
return _load_baseline(pl)[0]
|
||||
|
||||
|
||||
@@ -209,7 +209,7 @@ def _set_baseline_under_lock(
|
||||
meanwhile — lock the list and derive `dirty` from what is there, or the edit would read as
|
||||
synced and never reach YouTube. The name too: the lock re-reads a rename made meanwhile, and a
|
||||
baseline taken over THAT name matched it, so the list read clean with YouTube's title still the
|
||||
old one (E28 review round 4)."""
|
||||
old one."""
|
||||
_lock_or_gone(db, pl)
|
||||
_set_baseline(pl, ids, unavailable, name=name)
|
||||
recompute_dirty(db, pl)
|
||||
@@ -322,7 +322,7 @@ def user_sync_claim(db: Session, user_id: int) -> AbstractContextManager[bool]:
|
||||
def _mirror_listing(yt: YouTubeClient, ytid: str, count: object, pl_id: int) -> list[str] | None:
|
||||
"""A mirror's video ids from YouTube, or None when there is nothing to believe: the list was
|
||||
deleted between the pull's listing and this read (the next pull drops the mirror as vanished —
|
||||
not a reason to fail every other list, E32 review round 2), or YouTube listed no items for a
|
||||
not a reason to fail every other list), or YouTube listed no items for a
|
||||
list it counts as non-empty (`blank_listing`)."""
|
||||
try:
|
||||
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
||||
|
||||
@@ -187,7 +187,7 @@ def run_rss_poll(db: Session, channels: list[Channel] | None = None) -> dict[str
|
||||
# "fix this subscription"; `Connection refused` says "the egress is down".
|
||||
# The fallback's count rides along: a red card during an RSS outage should still say
|
||||
# whether the uploads are arriving another way.
|
||||
# As data too (E31): the audit log and the inbox say it in the reader's language, the first
|
||||
# As data too: the audit log and the inbox say it in the reader's language, the first
|
||||
# failure as its channel and its reason code (`http_404` vs `transport` is that difference).
|
||||
first_channel, first_reason = first_failure or ("", "error")
|
||||
raise progress.JobFailed(
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user