Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0528b35f2d | ||
|
|
cb46774851 | ||
|
|
f1cc6ce5dc | ||
|
|
4efe799eda | ||
|
|
40b27f2982 | ||
|
|
0314c4cce2 | ||
|
|
182ec31120 | ||
|
|
12a39f5291 | ||
|
|
79704b0001 | ||
|
|
4191189b4c | ||
|
|
7dbda60815 | ||
|
|
d1df76e8e5 | ||
|
|
3f6a9efd57 | ||
|
|
64e8f1da3d | ||
|
|
a1224e9a8c | ||
|
|
da4b19a39a | ||
|
|
5a74cf86b7 | ||
|
|
e432dc2780 | ||
|
|
3741d1a46d | ||
|
|
98ec1abf25 | ||
|
|
e5b155ada9 | ||
|
|
e09d613b04 | ||
|
|
38943fa93e | ||
|
|
887b4e546c | ||
|
|
df4a79a1f1 | ||
|
|
5d10a521a8 | ||
|
|
d7fb7075b8 | ||
|
|
a486c7521a | ||
|
|
76358a0d77 | ||
|
|
dc00b55feb | ||
|
|
fb6b56f121 | ||
|
|
2e2d6f5746 | ||
|
|
2ab9ff6a42 | ||
|
|
378c4daf0f | ||
|
|
8f1208fdfe | ||
|
|
52f12e5331 | ||
|
|
c4abbe48c1 | ||
|
|
4fc307c426 | ||
|
|
328d415cfe | ||
|
|
43e4929b45 | ||
|
|
5806adda7e | ||
|
|
c73761ae39 | ||
|
|
ca9ca85b18 | ||
|
|
614d728561 | ||
|
|
783f6408f9 | ||
|
|
751e1ae5ed | ||
|
|
da554df7c7 | ||
|
|
b1ff4058ac | ||
|
|
7ed2ac4e95 | ||
|
|
cff8fe9a6a | ||
|
|
471846d784 | ||
|
|
3c16012524 | ||
|
|
cdfe7a8986 | ||
|
|
3e96211db0 | ||
|
|
e9c15baae0 |
@@ -4,6 +4,7 @@
|
|||||||
*.env.local
|
*.env.local
|
||||||
**/__pycache__/
|
**/__pycache__/
|
||||||
**/*.pyc
|
**/*.pyc
|
||||||
|
**/.coverage
|
||||||
.venv/
|
.venv/
|
||||||
venv/
|
venv/
|
||||||
# Docker patterns are anchored at the context root with NO implicit `**/` (unlike .gitignore, where
|
# Docker patterns are anchored at the context root with NO implicit `**/` (unlike .gitignore, where
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
||||||
# independent of any one machine. Two jobs match the two local lanes:
|
# independent of any one machine. Two jobs match the two local lanes:
|
||||||
# frontend — tsc (3 projects) + eslint + prettier --check + knip + vitest
|
# frontend — tsc (3 projects) + eslint + prettier --check + knip (both modes) + vitest + vite build
|
||||||
# backend — ruff + the full pytest suite (pure-logic + the DB-backed lane) against a real Postgres
|
# + npm audit of the production dependencies
|
||||||
|
# backend — ruff + basedpyright + the full pytest suite (pure-logic + the DB-backed lane) against
|
||||||
|
# a real Postgres + pip-audit of the lock
|
||||||
#
|
#
|
||||||
# They are declared side by side but they DO NOT run in parallel: the lab has a single act_runner
|
# They are declared side by side but they DO NOT run in parallel: the lab has a single act_runner
|
||||||
# with one slot, so they queue. Measured across runs #86-#91 — the backend job starts at the exact
|
# with one slot, so they queue. Measured across runs #86-#91 — the backend job starts at the exact
|
||||||
@@ -56,6 +58,12 @@ jobs:
|
|||||||
- run: npm run format:check
|
- run: npm run format:check
|
||||||
- run: npm run knip
|
- run: npm run knip
|
||||||
- run: npm run test
|
- run: npm run test
|
||||||
|
# The production bundle (E32): before this, `vite build` — and devOnlyGuard, which fails it if
|
||||||
|
# dev-only code reaches the bundle — first ran inside the image build, after main was pushed.
|
||||||
|
- run: npx vite build
|
||||||
|
# What ships to the browser: a known CVE at high or above in a production dependency fails.
|
||||||
|
# (The dev tooling's own advisories are burned down by hand; see docs/gates.md.)
|
||||||
|
- run: npm audit --omit=dev --audit-level=high
|
||||||
|
|
||||||
backend:
|
backend:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -112,8 +120,9 @@ jobs:
|
|||||||
- name: pytest
|
- name: pytest
|
||||||
run: python -m pytest
|
run: python -m pytest
|
||||||
working-directory: backend
|
working-directory: backend
|
||||||
# Dependency CVE audit over the pinned lock. Non-blocking for now (informational) — a real CVE
|
# Dependency CVE audit over the pinned lock. BLOCKING since E32: it ran advisory-only, and when
|
||||||
# shows up in the run without failing the gate; tighten to a hard fail once the noise is known.
|
# it was finally read it had three urllib3 advisories with a fix available (bumped in the same
|
||||||
- name: pip-audit (advisory)
|
# change). An advisory without a fix is answered with `--ignore-vuln <id>` here, a reason and a
|
||||||
|
# date in the comment, and a line in docs/gates.md.
|
||||||
|
- name: pip-audit
|
||||||
run: pip install --no-cache-dir pip-audit && pip-audit -r backend/requirements.lock
|
run: pip install --no-cache-dir pip-audit && pip-audit -r backend/requirements.lock
|
||||||
continue-on-error: true
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ __pycache__/
|
|||||||
venv/
|
venv/
|
||||||
.mypy_cache/
|
.mypy_cache/
|
||||||
.pytest_cache/
|
.pytest_cache/
|
||||||
|
.coverage
|
||||||
.ruff_cache/
|
.ruff_cache/
|
||||||
|
|
||||||
# Node / frontend
|
# Node / frontend
|
||||||
|
|||||||
@@ -0,0 +1,15 @@
|
|||||||
|
# The CVE gate's allowlist (E32). The publish gate scans the built image and the committed tree with
|
||||||
|
# Trivy and fails on any HIGH or CRITICAL finding that has a fix available. A finding that cannot be
|
||||||
|
# fixed yet is let through ONLY by an entry here, and every entry carries both:
|
||||||
|
# statement — why it is acceptable for now (what is exposed, what is not)
|
||||||
|
# expired_at — the date it stops being accepted; after it, the gate fails again on its own
|
||||||
|
# A contract test (frontend/src/lib/gates.contract.test.ts) refuses an entry without either, and
|
||||||
|
# docs/gates.md lists them.
|
||||||
|
#
|
||||||
|
# Example:
|
||||||
|
# vulnerabilities:
|
||||||
|
# - id: CVE-2026-00000
|
||||||
|
# statement: in libfoo's XML path, which nothing in the image calls; no fixed Debian build yet
|
||||||
|
# expired_at: 2026-12-31
|
||||||
|
vulnerabilities: []
|
||||||
|
secrets: []
|
||||||
@@ -7,6 +7,25 @@ edit the release notes and re-run the generator.
|
|||||||
|
|
||||||
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
||||||
|
|
||||||
|
## 0.99.0 — 2026-10-04
|
||||||
|
|
||||||
|
### New
|
||||||
|
|
||||||
|
- Nearly every screen is now compiled by the React Compiler, so lists and menus re-render less.
|
||||||
|
- The channel list's row menu opens and closes without redrawing the table.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- A failed or empty answer from YouTube or Plex no longer empties a playlist, a mirror or your Plex watch history: Siftlode keeps what it had until a complete answer arrives.
|
||||||
|
- Plex watch states un-watched on Plex now sync for each library on its own, even if another library is briefly unreachable.
|
||||||
|
- A channel with no uploads no longer fails its backfill on every run.
|
||||||
|
- “This playlist no longer exists” appears only for a list you picked yourself.
|
||||||
|
- A filter on a deleted tag matches nothing and its chip says so, and the tag cleanup keeps tags a saved view filters on.
|
||||||
|
- A channel filter names its channels from the first moment, and a playlist reopened from the cache keeps its rows.
|
||||||
|
- The bug report opens on its title, and a dialog whose form loads late gives the form the focus once.
|
||||||
|
- The Plex backdrop crossfade setting is respected even with reduced motion on.
|
||||||
|
- Smaller fixes to keyboard focus, undo and loading states.
|
||||||
|
|
||||||
## 0.98.0 — 2026-10-03
|
## 0.98.0 — 2026-10-03
|
||||||
|
|
||||||
### New
|
### New
|
||||||
|
|||||||
@@ -12,12 +12,15 @@ router-free navigation provider.
|
|||||||
|
|
||||||
## The gate — never merge or ship red
|
## The gate — never merge or ship red
|
||||||
|
|
||||||
The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + typecheck (3 tsc projects) + eslint + prettier
|
The local gate is **ruff (`check` + `format --check`) + basedpyright (backend) + the lock check + typecheck (3 tsc projects) +
|
||||||
--check + knip + vitest + pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
eslint + prettier --check + knip (both modes) + vitest + vite build + pytest**, then a coverage report (no threshold). It runs
|
||||||
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
before every `release` merges and on every `publish`, which **refuses to build on failure — there is deliberately no `--skip`
|
||||||
|
flag**; a release also requires the Playwright suite, and the image must pass a Trivy scan before it is pushed. Run it and get it
|
||||||
|
green **before** any merge, release, or handing work to review.
|
||||||
|
|
||||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||||
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` + `ruff format --check backend` and `pytest` (the DB-lane needs Postgres).
|
- Directly, without the dispatcher: the commands in [`docs/development.md`](docs/development.md) § The gate (frontend lanes, `ruff` over `backend scripts`, basedpyright in the test image, and `pytest` — the DB-lane needs Postgres).
|
||||||
|
- **A gate with any real hit is never switched off or narrowed** — not a lint rule, a compiler flag, a lane or a release step. Fix the sites, or suppress each one inline naming its invariant (`-- why`, lint-enforced). What stays narrowed is in [`docs/gates.md`](docs/gates.md), and a contract test fails when a narrowing is missing from it. The effective ESLint config of every file and the CI/lane lines are pinned in `frontend/src/lib/gates.snapshot.json`: a new code file or any gate change fails until you regenerate it (`GATE_SNAPSHOT=update`, see gates.md) — then read that diff as a gate change. A comment saying "X is enforced by Y" is checked by running Y (E32: one such comment hid 95 functions the React Compiler skipped).
|
||||||
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` and burn down epic by epic. Fixed some? Regenerate with `--writebaseline`.
|
- **The backend type lane runs in the test image, not on the host** — `requirements.lock` cannot be installed on Windows (uvloop), and a type checker without the real packages reports nonsense. It fails on NEW errors only; the pre-existing ones sit in `backend/.basedpyright/baseline.json` and burn down epic by epic. Fixed some? Regenerate with `--writebaseline`.
|
||||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||||
|
|
||||||
@@ -77,37 +80,39 @@ CQRS, a repository/DI framework, a router library, a generic CRUD-router factory
|
|||||||
|
|
||||||
## Canonical patterns — when you have a choice, pick THIS
|
## Canonical patterns — when you have a choice, pick THIS
|
||||||
|
|
||||||
| Area | Use | Not |
|
| Area | Use | Not |
|
||||||
| ------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------ |
|
| ------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||||
| Request body | a Pydantic model | `payload: dict` |
|
| Request body | a Pydantic model | `payload: dict` |
|
||||||
| Request SIZE | nothing — `bodylimit` caps every route (SEC-20); need more? add a ceiling to `ROUTE_MAX_BODY_BYTES` | `capped_body` on a route that does not read raw bytes itself |
|
| Request SIZE | nothing — `bodylimit` caps every route (SEC-20); need more? add a ceiling to `ROUTE_MAX_BODY_BYTES` | `capped_body` on a route that does not read raw bytes itself |
|
||||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`), test-held | a bare `{"unsubscribed"}`/`{"cleared": n}`, no `ok` |
|
||||||
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
| Transaction commit | the handler commits; helpers `flush` (test-held list) | helpers that commit (only `quota` owns its session) |
|
||||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||||
| FE button | `ui/Button` (a `SIZES` rung) · `IconButton` · `Pressable` (a clickable card/row) · `Chip` · `DragHandle` | a raw `<button>` or `FocusSafeButton` outside the control primitives (lint-enforced) |
|
| FE button | `ui/Button` (a `SIZES` rung) · `IconButton` · `Pressable` (a clickable card/row) · `Chip` · `DragHandle` | a raw `<button>` or `FocusSafeButton` outside the control primitives (lint-enforced) |
|
||||||
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
| FE keyboard shortcut | a row in `lib/keymap.ts` + `dispatch()` | a private `keydown` listener with its own key literals |
|
||||||
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
| FE list/grid keys | `NavGrid`/`keyboardNav` + `navItemAttrs` + `data-nav-open` | a private arrow-key / roving-tabindex handler |
|
||||||
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
| FE view push (history) | `rememberNavPosition()` right before `history.pushState` | a push that forgets: Back strands focus on `<body>` |
|
||||||
| FE text field / checkbox / slider | `ui/Input` · `Textarea` · `InnerInput` in a `fieldClass` frame · `Checkbox`/`CheckRow` · `Slider` | a raw `<input>`/`<textarea>` anywhere but those primitives (lint-enforced) |
|
| FE text field / checkbox / slider | `ui/Input` · `Textarea` · `InnerInput` in a `fieldClass` frame · `Checkbox`/`CheckRow` · `Slider` | a raw `<input>`/`<textarea>` anywhere but those primitives (lint-enforced) |
|
||||||
| FE glass surface | a role: `SURFACE.panel` / `card` / `menu` / `popover` / `tooltip` (`ui/surfaces.ts`) | a bare `glass`/`glass-card`/`glass-menu` beside a hand-picked radius (lint-enforced) |
|
| FE glass surface | a role: `SURFACE.panel` / `card` / `menu` / `popover` / `tooltip` (`ui/surfaces.ts`) | a bare `glass`/`glass-card`/`glass-menu` beside a hand-picked radius (lint-enforced) |
|
||||||
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
| FE static label / button-like link | `Badge` (flat, no edge) · `LinkButton` / `IconLink` | a glass pill that reads as a chip; an `<a>` hand-painted like a button |
|
||||||
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
| FE control | `SegmentedToggle`; a long or user's list: `FilterSelect` | a native `<select>` (lint-enforced) |
|
||||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||||
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
| FE failed write | `meta.failTitle`, else `notifyActionError(err, title)` | a bare `catch {}` or `notify()` with no error attached |
|
||||||
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
| FE network call | `req`/`uploadReq`/`beacon`, or `lib/http.ts` `send` (public pages) | a bare `fetch`/`XMLHttpRequest`/`WebSocket` (lint-enforced) |
|
||||||
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
| FE diagnostics | `lib/log.ts` (`log.info("event.code", msg, attrs)`); a filter change → `lib/filterLog.ts`, a player → `lib/mediaLog.ts` | `console.*` (lint-enforced); a value only a higher level may carry, logged unguarded |
|
||||||
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
| FE state that MIRRORS a prop/query | derive it, or hold a draft (`draft ?? server`) | a copy an effect re-syncs — that paints the stale value first |
|
||||||
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
| FE state reset on a key change | adjust it DURING render against a remembered key | `useEffect(() => setX(init), [key])` |
|
||||||
| FE effect that must not re-subscribe | `useEffectEvent` (React 19.2+), else a latest-value ref | an excused dep list asserting the same thing in prose |
|
| FE effect that must not re-subscribe | `useEffectEvent` (React 19.2+); a handler or timer: `useLatest` / `useStableCallback` | a render-phase `ref.current = x` — the React Compiler skips the component |
|
||||||
| FE hook suppression | an inline `eslint-disable-next-line` NAMING the invariant | `eslint-suppressions.json` (deliberately absent — E27 S2) |
|
| FE hook suppression | an inline `eslint-disable-next-line` NAMING the invariant | `eslint-suppressions.json` (deliberately absent — E27 S2) |
|
||||||
|
| FE popover / scroll-fade ref | destructure it: `const { setReference } = pop.refs`, then `ref={setReference}` | `ref={pop.refs.setReference}` / `ref={fade.ref}` — the compiler then treats the whole object as a ref and skips |
|
||||||
|
| FE async handler with cleanup | `try { … } catch { … }`, then the cleanup after it (steps that `return` early go in their own function); a conditional, loop or `?.` the `try` needs goes before it or into a module-level helper | `try/finally`, or a `?:` / `??` / loop / `?.` inside `try` — the React Compiler skips the whole component (and `react-hooks/todo` misses most of these) |
|
||||||
|
|
||||||
## House rules that bite (learned from real bugs)
|
## House rules that bite (learned from real bugs)
|
||||||
|
|
||||||
|
|||||||
+7
-1
@@ -53,8 +53,11 @@ RUN adduser --disabled-password --gecos "" appuser \
|
|||||||
# Harmless on hosts with no /dev/dri (the driver just goes unused). Deno: the JavaScript runtime yt-dlp
|
# Harmless on hosts with no /dev/dri (the driver just goes unused). Deno: the JavaScript runtime yt-dlp
|
||||||
# uses to solve YouTube's "n" signature challenge for the web player clients (required alongside the PO
|
# uses to solve YouTube's "n" signature challenge for the web player clients (required alongside the PO
|
||||||
# token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
# token, else only storyboards are offered). yt-dlp auto-detects deno on PATH.
|
||||||
|
# `apt-get upgrade` (E32): the base image is a snapshot, and the Debian security fixes released since it
|
||||||
|
# was cut are only in the archive — the first Trivy scan found one (pcre2) fixed upstream but not in it.
|
||||||
ARG DENO_VERSION=v2.9.1
|
ARG DENO_VERSION=v2.9.1
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
|
&& apt-get upgrade -y --no-install-recommends \
|
||||||
&& apt-get install -y --no-install-recommends ffmpeg intel-media-va-driver unzip curl ca-certificates \
|
&& apt-get install -y --no-install-recommends ffmpeg intel-media-va-driver unzip curl ca-certificates \
|
||||||
&& curl -fsSL "https://github.com/denoland/deno/releases/download/${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o /tmp/deno.zip \
|
&& curl -fsSL "https://github.com/denoland/deno/releases/download/${DENO_VERSION}/deno-x86_64-unknown-linux-gnu.zip" -o /tmp/deno.zip \
|
||||||
&& unzip -o /tmp/deno.zip -d /usr/local/bin \
|
&& unzip -o /tmp/deno.zip -d /usr/local/bin \
|
||||||
@@ -78,9 +81,12 @@ RUN printf 'Types: deb\nURIs: http://deb.debian.org/debian\nSuites: bookworm\nCo
|
|||||||
# Install from the PINNED lock (requirements.lock, generated from requirements.txt) so the image is
|
# Install from the PINNED lock (requirements.lock, generated from requirements.txt) so the image is
|
||||||
# byte-reproducible and the exact versions are the ones pip-audit scanned in CI — not whatever floats
|
# byte-reproducible and the exact versions are the ones pip-audit scanned in CI — not whatever floats
|
||||||
# up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
# up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
||||||
|
# pip itself is removed once it has installed the lock (E32): nothing runs it at runtime, and it carries
|
||||||
|
# its own VENDORED copies of urllib3, msgpack and setuptools, which Trivy reports as the image's own —
|
||||||
|
# an attack surface and a CVE stream for a tool the running app never calls.
|
||||||
COPY backend/requirements.lock .
|
COPY backend/requirements.lock .
|
||||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||||
pip install --upgrade pip && pip install -r requirements.lock
|
pip install --upgrade pip && pip install -r requirements.lock && pip uninstall -y pip
|
||||||
|
|
||||||
COPY --chown=appuser backend/ .
|
COPY --chown=appuser backend/ .
|
||||||
# Read at runtime by config.py as the app_version (a committed single source of truth).
|
# Read at runtime by config.py as the app_version (a committed single source of truth).
|
||||||
|
|||||||
@@ -548,64 +548,6 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"./app/plex/stream.py": [
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 51,
|
|
||||||
"endColumn": 55,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 23,
|
|
||||||
"endColumn": 25,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 18,
|
|
||||||
"endColumn": 31,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 75,
|
|
||||||
"endColumn": 79,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 10,
|
|
||||||
"endColumn": 23,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 10,
|
|
||||||
"endColumn": 12,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportOptionalMemberAccess",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 10,
|
|
||||||
"endColumn": 17,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"./app/plex/sync.py": [
|
"./app/plex/sync.py": [
|
||||||
{
|
{
|
||||||
"code": "reportArgumentType",
|
"code": "reportArgumentType",
|
||||||
@@ -638,14 +580,6 @@
|
|||||||
"endColumn": 89,
|
"endColumn": 89,
|
||||||
"lineCount": 1
|
"lineCount": 1
|
||||||
}
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 17,
|
|
||||||
"endColumn": 18,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"./app/plex/tmdb.py": [
|
"./app/plex/tmdb.py": [
|
||||||
@@ -1650,24 +1584,6 @@
|
|||||||
"endColumn": 36,
|
"endColumn": 36,
|
||||||
"lineCount": 1
|
"lineCount": 1
|
||||||
}
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
"code": "reportArgumentType",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 46,
|
|
||||||
"endColumn": 52,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"./app/youtube/rss.py": [
|
|
||||||
{
|
|
||||||
"code": "reportGeneralTypeIssues",
|
|
||||||
"range": {
|
|
||||||
"startColumn": 10,
|
|
||||||
"endColumn": 15,
|
|
||||||
"lineCount": 1
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,7 +72,9 @@ mutation it records, so an audit entry can never exist without its change (or vi
|
|||||||
## The gate
|
## The gate
|
||||||
|
|
||||||
Everything below must be green before a change is done (run via `siftlode check`):
|
Everything below must be green before a change is done (run via `siftlode check`):
|
||||||
`ruff` (backend) · `tsc` ×3 + `eslint` + `prettier` + `knip` (frontend) · `vitest` · `pytest`. The
|
`ruff` + `basedpyright` (backend) · the lock check + `tsc` ×3 + `eslint` + `prettier` + `knip` +
|
||||||
pytest lane runs in `backend/docker-compose.test.yml` — an ephemeral Postgres + the backend tree
|
`vite build` (frontend) · `vitest` · `pytest`. The pytest lane runs in `backend/docker-compose.test.yml`
|
||||||
bind-mounted — so the DB-backed integration tests (the `db` fixture) actually run; `SIFTLODE_TEST_
|
— an ephemeral Postgres + the backend tree bind-mounted — so the DB-backed integration tests (the `db`
|
||||||
REQUIRE_DB=1` there turns a would-be silent skip into a hard failure. See `docs/development.md`.
|
fixture) actually run; a missing database is a hard failure, not a silent skip, unless a hand run asks
|
||||||
|
for pure-logic only with `SIFTLODE_TEST_REQUIRE_DB=0`. See `docs/development.md`, and `docs/gates.md`
|
||||||
|
for what each gate deliberately leaves out.
|
||||||
|
|||||||
@@ -219,7 +219,7 @@ class BodyLimitMiddleware:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
await self.app(scope, counting_receive, guarded_send)
|
await self.app(scope, counting_receive, guarded_send)
|
||||||
except Exception: # noqa: BLE001 — only swallowed when WE caused it, see below
|
except Exception: # only swallowed when WE caused it, see below
|
||||||
# ClientDisconnect (or whatever a handler raises on a truncated read) is the expected
|
# ClientDisconnect (or whatever a handler raises on a truncated read) is the expected
|
||||||
# consequence of the disconnect we just faked, and the caller must get the 413 rather
|
# consequence of the disconnect we just faked, and the caller must get the 413 rather
|
||||||
# than a 500. Anything raised when we did NOT refuse is a real failure and is re-raised
|
# than a 500. Anything raised when we did NOT refuse is a real failure and is re-raised
|
||||||
|
|||||||
@@ -400,7 +400,7 @@ def escalate_kill(proc: subprocess.Popen, terminate, kill, grace_s: float, label
|
|||||||
try:
|
try:
|
||||||
proc.wait(timeout=grace_s)
|
proc.wait(timeout=grace_s)
|
||||||
except subprocess.TimeoutExpired:
|
except subprocess.TimeoutExpired:
|
||||||
# noqa TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
# TRY400: the exception IS the message here. A TimeoutExpired's traceback is this
|
||||||
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
# `wait` call and nothing else — it would add three frames and no fact the line lacks.
|
||||||
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
log.error("%s pid %s survived SIGKILL", label, proc.pid) # noqa: TRY400
|
||||||
|
|
||||||
@@ -456,7 +456,7 @@ def run_ffmpeg(
|
|||||||
try:
|
try:
|
||||||
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
|
ret = proc.wait(timeout=_FFMPEG_EXIT_GRACE_S)
|
||||||
except subprocess.TimeoutExpired: # pipes closed but the process lingers
|
except subprocess.TimeoutExpired: # pipes closed but the process lingers
|
||||||
raise StatedFailure("ffmpeg did not exit after closing its output")
|
raise StatedFailure("ffmpeg did not exit after closing its output") from None
|
||||||
except BaseException:
|
except BaseException:
|
||||||
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
|
# ONE exit path stops the process, so the invariant is unconditional: nothing leaves this
|
||||||
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
|
# function with ffmpeg still running. Not just our own EditAborted/stall — `on_progress`
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ def main() -> None:
|
|||||||
logs.configure_logging("runner")
|
logs.configure_logging("runner")
|
||||||
_deprioritize()
|
_deprioritize()
|
||||||
if len(sys.argv) != 2:
|
if len(sys.argv) != 2:
|
||||||
# noqa T201: a usage line is for the person who typed the command. Logging is already
|
# T201: a usage line is for the person who typed the command. Logging is already
|
||||||
# configured here, and in a deployment it writes JSON records — the one reader of this
|
# configured here, and in a deployment it writes JSON records — the one reader of this
|
||||||
# line wants the bare sentence, not a record with a request id they do not have.
|
# line wants the bare sentence, not a record with a request id they do not have.
|
||||||
print( # noqa: T201
|
print( # noqa: T201
|
||||||
@@ -104,7 +104,7 @@ def _run(job_id, asset_id, job_kind, source_kind, source_ref, spec, user_id) ->
|
|||||||
else:
|
else:
|
||||||
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
_download(job_id, asset_id, source_kind, source_ref, spec, user_id)
|
||||||
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
except Exception as exc: # noqa: BLE001 — whatever it is, it is reported here or raw
|
||||||
# noqa TRY400: the stack goes with it scrubbed (`scrubbed_stack`); `exc_info` is raw.
|
# TRY400: the stack goes with it scrubbed (`scrubbed_stack`); `exc_info` is raw.
|
||||||
log.error( # noqa: TRY400
|
log.error( # noqa: TRY400
|
||||||
"the download child for job %s failed",
|
"the download child for job %s failed",
|
||||||
job_id,
|
job_id,
|
||||||
|
|||||||
@@ -106,15 +106,30 @@ def _tag_visible(user: User):
|
|||||||
return or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
return or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||||
|
|
||||||
|
|
||||||
def tags_by_category(db: Session, tag_ids: list[int]) -> dict[str, list[int]]:
|
# The pseudo-category of a selected tag id no Tag row answers (a saved view or the feed state outlives
|
||||||
|
# a deleted tag). It filters like a category whose tags no channel carries — to nothing, in the SQL and
|
||||||
|
# in feed_summary's mirror alike — instead of the id being dropped, which showed the WHOLE feed under
|
||||||
|
# the view's old name (E32 S2). No real category is called this (language/topic/other).
|
||||||
|
MISSING_TAG_CATEGORY = "missing"
|
||||||
|
|
||||||
|
|
||||||
|
def tags_by_category(db: Session, user: User, tag_ids: list[int]) -> dict[str, list[int]]:
|
||||||
"""Group tag ids by their category (language/topic/other). The single source for the category split
|
"""Group tag ids by their category (language/topic/other). The single source for the category split
|
||||||
shared by the feed's tag WHERE (filtered_query) and the facet/count summary (feed_summary), so the
|
shared by the feed's tag WHERE (filtered_query) and the facet/count summary (feed_summary), so the
|
||||||
two can't disagree on which selected tag belongs to which category."""
|
two can't disagree on which selected tag belongs to which category. An unknown id lands in
|
||||||
|
MISSING_TAG_CATEGORY — and so does another user's private tag: resolving it to its real category
|
||||||
|
told the caller which foreign ids exist (E32 review)."""
|
||||||
if not tag_ids:
|
if not tag_ids:
|
||||||
return {} # skip the pointless `WHERE id IN ()` round-trip on the no-tag-filter hot path
|
return {} # skip the pointless `WHERE id IN ()` round-trip on the no-tag-filter hot path
|
||||||
out: dict[str, list[int]] = {}
|
out: dict[str, list[int]] = {}
|
||||||
for tid, cat in db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids))).all():
|
visible = or_(Tag.user_id.is_(None), Tag.user_id == user.id)
|
||||||
|
for tid, cat in db.execute(
|
||||||
|
select(Tag.id, Tag.category).where(Tag.id.in_(tag_ids), visible)
|
||||||
|
).all():
|
||||||
out.setdefault(cat, []).append(tid)
|
out.setdefault(cat, []).append(tid)
|
||||||
|
known = {tid for ids in out.values() for tid in ids}
|
||||||
|
if missing := [tid for tid in tag_ids if tid not in known]:
|
||||||
|
out[MISSING_TAG_CATEGORY] = missing
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
@@ -304,7 +319,7 @@ def filtered_query(
|
|||||||
# ALL its selected tags. ⚠️ feed_summary.passes() reimplements this exact rule in Python for the
|
# ALL its selected tags. ⚠️ feed_summary.passes() reimplements this exact rule in Python for the
|
||||||
# sidebar count/facets — keep the two in lockstep (equivalence-tested by
|
# sidebar count/facets — keep the two in lockstep (equivalence-tested by
|
||||||
# test_feed_summary_matches_count_plus_facets).
|
# test_feed_summary_matches_count_plus_facets).
|
||||||
by_category = tags_by_category(db, tags)
|
by_category = tags_by_category(db, user, tags)
|
||||||
# Facet counting drops the category being counted so its own chips don't zero each
|
# Facet counting drops the category being counted so its own chips don't zero each
|
||||||
# other out (standard drill-down faceting: a category's count ignores its own
|
# other out (standard drill-down faceting: a category's count ignores its own
|
||||||
# selections but still honours the other categories' filters).
|
# selections but still honours the other categories' filters).
|
||||||
@@ -395,7 +410,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
|||||||
return {"count": 0, "facets": {}}
|
return {"count": 0, "facets": {}}
|
||||||
|
|
||||||
# channel -> its visible tag ids, plus each tag's category (for grouping the facet output).
|
# channel -> its visible tag ids, plus each tag's category (for grouping the facet output).
|
||||||
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
visible = _tag_visible(user)
|
||||||
channel_tags: dict[str, set[int]] = {}
|
channel_tags: dict[str, set[int]] = {}
|
||||||
tag_category: dict[int, str] = {}
|
tag_category: dict[int, str] = {}
|
||||||
for cid, tid, cat in db.execute(
|
for cid, tid, cat in db.execute(
|
||||||
@@ -410,7 +425,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
|||||||
# base channel carries still filters everything out for its category — matching the SQL `IN (empty)`
|
# base channel carries still filters everything out for its category — matching the SQL `IN (empty)`
|
||||||
# behaviour).
|
# behaviour).
|
||||||
selected_by_cat: dict[str, set[int]] = {
|
selected_by_cat: dict[str, set[int]] = {
|
||||||
cat: set(ids) for cat, ids in tags_by_category(db, params.get("tags") or []).items()
|
cat: set(ids) for cat, ids in tags_by_category(db, user, params.get("tags") or []).items()
|
||||||
}
|
}
|
||||||
|
|
||||||
def passes(ch_tags: set[int], exclude_cat: str | None) -> bool:
|
def passes(ch_tags: set[int], exclude_cat: str | None) -> bool:
|
||||||
@@ -438,7 +453,7 @@ def feed_summary(db: Session, user: User, **params) -> dict:
|
|||||||
# Drop this category's own selections so its chips keep independent counts; a conjunctive
|
# Drop this category's own selections so its chips keep independent counts; a conjunctive
|
||||||
# topic-AND keeps them (each remaining chip narrows to channels that ALSO have the selected).
|
# topic-AND keeps them (each remaining chip narrows to channels that ALSO have the selected).
|
||||||
exclude = None if tag_modes.get(category, "or") == "and" else category
|
exclude = None if tag_modes.get(category, "or") == "and" else category
|
||||||
for cid, ch_tags in channel_tags.items():
|
for ch_tags in channel_tags.values():
|
||||||
if not passes(ch_tags, exclude):
|
if not passes(ch_tags, exclude):
|
||||||
continue
|
continue
|
||||||
for tid in ch_tags:
|
for tid in ch_tags:
|
||||||
@@ -671,8 +686,8 @@ def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str, datetime |
|
|||||||
raw = payload["k"]
|
raw = payload["k"]
|
||||||
if len(raw) != len(keys):
|
if len(raw) != len(keys):
|
||||||
raise ValueError("key arity mismatch")
|
raise ValueError("key arity mismatch")
|
||||||
vals = [_cursor_dt(v) if k["kind"] == "dt" else v for k, v in zip(keys, raw)]
|
vals = [_cursor_dt(v) if k["kind"] == "dt" else v for k, v in zip(keys, raw, strict=True)]
|
||||||
for k, v in zip(keys, vals):
|
for k, v in zip(keys, vals, strict=True):
|
||||||
if v is None and not k["nullable"]:
|
if v is None and not k["nullable"]:
|
||||||
raise ValueError("null value for a non-nullable key")
|
raise ValueError("null value for a non-nullable key")
|
||||||
window = payload.get("w")
|
window = payload.get("w")
|
||||||
|
|||||||
@@ -239,7 +239,7 @@ def deliver(
|
|||||||
# already open on the tracker and this is what would have recorded its link. We
|
# already open on the tracker and this is what would have recorded its link. We
|
||||||
# still must not write — another send owns the row and is opening its own issue —
|
# still must not write — another send owns the row and is opening its own issue —
|
||||||
# so say which issue was left behind, because nothing else will.
|
# so say which issue was left behind, because nothing else will.
|
||||||
# noqa TRY400: the traceback is one SendClaimLost carrying the session id and
|
# TRY400: the traceback is one SendClaimLost carrying the session id and
|
||||||
# nothing else; the issue number below is the whole news.
|
# nothing else; the issue number below is the whole news.
|
||||||
log.error( # noqa: TRY400
|
log.error( # noqa: TRY400
|
||||||
"an issue was created by a send that had already lost its claim",
|
"an issue was created by a send that had already lost its claim",
|
||||||
|
|||||||
+1
-1
@@ -166,7 +166,7 @@ def bind(**fields: Any):
|
|||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
for key, token in zip(keys, tokens):
|
for key, token in zip(keys, tokens, strict=True):
|
||||||
_CONTEXT_VARS[key].reset(token)
|
_CONTEXT_VARS[key].reset(token)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+21
-25
@@ -1,38 +1,36 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import math
|
import math
|
||||||
|
import mimetypes
|
||||||
from contextlib import asynccontextmanager, suppress
|
from contextlib import asynccontextmanager, suppress
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from fastapi import FastAPI, HTTPException, Request
|
from fastapi import FastAPI, HTTPException, Request
|
||||||
from fastapi.encoders import jsonable_encoder
|
from fastapi.encoders import jsonable_encoder
|
||||||
from fastapi.exceptions import RequestValidationError
|
from fastapi.exceptions import RequestValidationError
|
||||||
|
|
||||||
# Logging is configured HERE, before the route modules are imported, so every logger they create at
|
|
||||||
# import time is already covered. ONE format for the api, the worker, the download child and alembic
|
|
||||||
# alike (app/logs.py) — this replaces the old uvicorn --log-config + the hand-rolled fallback that
|
|
||||||
# used to live here, and uvicorn's access log is replaced by the `http.request` middleware below.
|
|
||||||
from app import logs
|
|
||||||
from app.logs import configure_logging, current_level, log_requests, set_level
|
|
||||||
|
|
||||||
configure_logging("api")
|
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
|
||||||
from fastapi.staticfiles import StaticFiles
|
from fastapi.staticfiles import StaticFiles
|
||||||
from sqlalchemy.exc import DataError
|
from sqlalchemy.exc import DataError
|
||||||
from starlette.middleware.sessions import SessionMiddleware
|
from starlette.middleware.sessions import SessionMiddleware
|
||||||
|
|
||||||
from app import bodylimit
|
# Logging is configured HERE, before the app's own modules are imported, so a line one of them logs
|
||||||
from app import collation
|
# WHILE importing already has the format (a logger is configured whenever it is created —
|
||||||
from app import logsink
|
# app/logs.py — but a line emitted before `configure_logging` goes out raw). ONE format for the api,
|
||||||
from app import state
|
# the worker, the download child and alembic alike — this replaces the old uvicorn --log-config +
|
||||||
from app import sysconfig
|
# the hand-rolled fallback that used to live here, and uvicorn's access log is replaced by the
|
||||||
from app.config import settings
|
# `http.request` middleware below. The imports after it are E402 on purpose, each one marked.
|
||||||
from app.db import SessionLocal, engine
|
from app import logs
|
||||||
from app.plex import stream as plex_stream
|
from app.logs import configure_logging, current_level, log_requests, set_level
|
||||||
from app.plex import subtitles as plex_subtitles
|
|
||||||
from app.routes import (
|
configure_logging("api")
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
from app import bodylimit, collation, logsink, state, sysconfig # noqa: E402 — logging is configured first
|
||||||
|
from app.config import settings # noqa: E402 — logging is configured first
|
||||||
|
from app.db import SessionLocal, engine # noqa: E402 — logging is configured first
|
||||||
|
from app.plex import stream as plex_stream # noqa: E402 — logging is configured first
|
||||||
|
from app.plex import subtitles as plex_subtitles # noqa: E402 — logging is configured first
|
||||||
|
from app.routes import ( # noqa: E402 — logging is configured first
|
||||||
admin,
|
admin,
|
||||||
admin_logs as admin_logs_routes,
|
admin_logs as admin_logs_routes,
|
||||||
audit as audit_routes,
|
audit as audit_routes,
|
||||||
@@ -60,7 +58,7 @@ from app.routes import (
|
|||||||
tags,
|
tags,
|
||||||
version,
|
version,
|
||||||
)
|
)
|
||||||
from app.scheduler import shutdown_scheduler, start_scheduler
|
from app.scheduler import shutdown_scheduler, start_scheduler # noqa: E402 — logging is configured first
|
||||||
|
|
||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
@@ -77,7 +75,7 @@ async def lifespan(app: FastAPI):
|
|||||||
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
# booting — an unreachable or not-yet-migrated database has to start into the env level.
|
||||||
try:
|
try:
|
||||||
set_level(sysconfig.get_str(db, "log_level"))
|
set_level(sysconfig.get_str(db, "log_level"))
|
||||||
except Exception: # noqa: BLE001 — any DB/config failure keeps the bootstrap level
|
except Exception: # any DB/config failure keeps the bootstrap level
|
||||||
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
# With the traceback: this catch spans a connection, a query and a coercion at boot,
|
||||||
# and `%s` of the exception does not say which of them raised.
|
# and `%s` of the exception does not say which of them raised.
|
||||||
log.exception("could not read the stored log level; staying at %s", current_level())
|
log.exception("could not read the stored log level; staying at %s", current_level())
|
||||||
@@ -332,8 +330,6 @@ app.include_router(setup_routes.router)
|
|||||||
|
|
||||||
# Ensure modern image types resolve to the right Content-Type when FileResponse guesses from the
|
# Ensure modern image types resolve to the right Content-Type when FileResponse guesses from the
|
||||||
# filename (the runtime's mimetypes db doesn't always know .webp → it'd fall back to octet-stream).
|
# filename (the runtime's mimetypes db doesn't always know .webp → it'd fall back to octet-stream).
|
||||||
import mimetypes
|
|
||||||
|
|
||||||
mimetypes.add_type("image/webp", ".webp")
|
mimetypes.add_type("image/webp", ".webp")
|
||||||
mimetypes.add_type("image/avif", ".avif")
|
mimetypes.add_type("image/avif", ".avif")
|
||||||
|
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from sqlalchemy.dialects.postgresql import JSONB, TSVECTOR
|
|||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||||
|
|
||||||
from app.db import Base
|
from app.db import Base
|
||||||
|
from app.utils import canonical_email
|
||||||
|
|
||||||
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
||||||
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
||||||
@@ -51,9 +52,6 @@ def display_text(raw: str | None) -> str | None:
|
|||||||
return (raw or "").strip()[:DISPLAY_TEXT_MAX] or None
|
return (raw or "").strip()[:DISPLAY_TEXT_MAX] or None
|
||||||
|
|
||||||
|
|
||||||
from app.utils import canonical_email
|
|
||||||
|
|
||||||
|
|
||||||
# The resume-position columns (VideoState / PlexState.position_seconds) are int4. The checkpoint
|
# The resume-position columns (VideoState / PlexState.position_seconds) are int4. The checkpoint
|
||||||
# routes bound their bodies by this, so an out-of-range position is a 422 rather than a 500.
|
# routes bound their bodies by this, so an out-of-range position is a 422 rather than a 500.
|
||||||
POSITION_MAX_S = 2**31 - 1
|
POSITION_MAX_S = 2**31 - 1
|
||||||
|
|||||||
@@ -78,9 +78,10 @@ class PlexClient:
|
|||||||
# --- Catalog mirror (P1) ---
|
# --- Catalog mirror (P1) ---
|
||||||
def section_items(
|
def section_items(
|
||||||
self, key: str, item_type: int, start: int = 0, size: int = 200
|
self, key: str, item_type: int, start: int = 0, size: int = 200
|
||||||
) -> tuple[list[dict], int]:
|
) -> tuple[list[dict], int | None]:
|
||||||
"""A page of a section's top-level items. type 1 = movie, 2 = show. Returns
|
"""A page of a section's top-level items. type 1 = movie, 2 = show. Returns
|
||||||
(items, total_size)."""
|
(items, total_size); the total is None when the answer carries none, so a container that
|
||||||
|
says "0" (a genuinely empty section) stays distinguishable from one that says nothing."""
|
||||||
mc = self._get(
|
mc = self._get(
|
||||||
f"/library/sections/{key}/all",
|
f"/library/sections/{key}/all",
|
||||||
params={
|
params={
|
||||||
@@ -93,7 +94,10 @@ class PlexClient:
|
|||||||
"X-Plex-Container-Size": size,
|
"X-Plex-Container-Size": size,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
return mc.get("Metadata", []) or [], int(mc.get("totalSize", mc.get("size", 0)) or 0)
|
# `totalSize` only: `size` is THIS page's count, and read as the total it made a single
|
||||||
|
# page look like the whole section — "complete" to a prune or a reconcile (E32 review).
|
||||||
|
total = mc.get("totalSize")
|
||||||
|
return mc.get("Metadata", []) or [], None if total is None else int(total)
|
||||||
|
|
||||||
def children(self, rating_key: str) -> list[dict]:
|
def children(self, rating_key: str) -> list[dict]:
|
||||||
"""Direct children (show → seasons, season → episodes)."""
|
"""Direct children (show → seasons, season → episodes)."""
|
||||||
@@ -136,9 +140,14 @@ class PlexClient:
|
|||||||
# --- Collection editing (P2 — write-back to Plex; all shared/global, admin-gated in the routes) ---
|
# --- Collection editing (P2 — write-back to Plex; all shared/global, admin-gated in the routes) ---
|
||||||
@property
|
@property
|
||||||
def machine_id(self) -> str:
|
def machine_id(self) -> str:
|
||||||
"""Server machineIdentifier — needed in the `uri` of collection add/create calls. Cached."""
|
"""Server machineIdentifier — needed in the `uri` of collection add/create calls. Cached.
|
||||||
|
Without one the uri would read `server:///…`, a malformed write Plex answers in its own way;
|
||||||
|
it is refused here, by name (E32 S2)."""
|
||||||
if not getattr(self, "_machine_id", None):
|
if not getattr(self, "_machine_id", None):
|
||||||
self._machine_id = self._get("/identity").get("machineIdentifier") or ""
|
mid = self._get("/identity").get("machineIdentifier")
|
||||||
|
if not mid:
|
||||||
|
raise PlexError("Plex did not report its machine identifier")
|
||||||
|
self._machine_id = mid
|
||||||
return self._machine_id
|
return self._machine_id
|
||||||
|
|
||||||
def _write(self, method: str, path: str) -> dict:
|
def _write(self, method: str, path: str) -> dict:
|
||||||
|
|||||||
@@ -0,0 +1,150 @@
|
|||||||
|
"""The HU overlay: a Hungarian Plex section written onto its English twin's rows (E30), split out of
|
||||||
|
`plex/sync.py` (E32 S2), which runs it once per paired section during the full sync.
|
||||||
|
|
||||||
|
The pairing is admin-configured (`plex_overlay_map`, ``huKey:enKey,...``). An overlay section creates
|
||||||
|
no browsable rows of its own; it only fills the `*_hu` columns of the EN rows it matches by guid.
|
||||||
|
`plex/sync.py` and this module import each other as MODULES (attribute access at call time), which is
|
||||||
|
what lets either be imported first.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app import logs, sysconfig
|
||||||
|
from app.models import PlexItem, PlexLibrary, PlexShow
|
||||||
|
from app.plex import sync as plex_sync
|
||||||
|
from app.plex.client import PlexClient
|
||||||
|
from app.plex.genres import hu_genres
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def overlay_map(db: Session) -> dict[str, str]:
|
||||||
|
"""The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
|
||||||
|
('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
|
||||||
|
Malformed pairs are skipped."""
|
||||||
|
raw = sysconfig.get_str(db, "plex_overlay_map")
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
for pair in raw.split(","):
|
||||||
|
hu, sep, en = pair.partition(":")
|
||||||
|
hu, en = hu.strip(), en.strip()
|
||||||
|
if sep and hu and en:
|
||||||
|
out[hu] = en
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def sync_overlay(
|
||||||
|
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
||||||
|
) -> None:
|
||||||
|
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
||||||
|
NO browsable rows — an overlay library is never listed on its own. Only title/summary/tagline/genres
|
||||||
|
differ by language; the language-neutral people_text feeds both search vectors. A show section's
|
||||||
|
episodes are overlaid too (E30): the HU section lists every episode with its Hungarian title and
|
||||||
|
summary, and the episode rows live in the EN show library.
|
||||||
|
|
||||||
|
This is a full RECONCILE: after a COMPLETE HU listing every guid-bearing EN row's HU fields are
|
||||||
|
cleared, then the matches are repopulated — so an EN row whose HU twin vanished (or was re-guid'd)
|
||||||
|
falls back to EN instead of keeping stale Hungarian metadata. A listing cut short only adds. HU
|
||||||
|
items with no EN match are skipped."""
|
||||||
|
if lib.kind == "movie":
|
||||||
|
passes = [
|
||||||
|
(
|
||||||
|
plex_sync._TYPE_MOVIE,
|
||||||
|
select(PlexItem).where(
|
||||||
|
PlexItem.library_id == en_lib_id,
|
||||||
|
PlexItem.kind == "movie",
|
||||||
|
PlexItem.guid.is_not(None),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
passes = [
|
||||||
|
(
|
||||||
|
plex_sync._TYPE_SHOW,
|
||||||
|
select(PlexShow).where(
|
||||||
|
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
plex_sync._TYPE_EPISODE,
|
||||||
|
select(PlexItem).where(
|
||||||
|
PlexItem.library_id == en_lib_id,
|
||||||
|
PlexItem.kind == "episode",
|
||||||
|
PlexItem.guid.is_not(None),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for item_type, rows_q in passes:
|
||||||
|
_overlay_pass(db, plex, lib, item_type, db.execute(rows_q).scalars(), stats)
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _overlay_pass(
|
||||||
|
db: Session,
|
||||||
|
plex: PlexClient,
|
||||||
|
lib: PlexLibrary,
|
||||||
|
item_type: int,
|
||||||
|
rows: Iterable[PlexItem] | Iterable[PlexShow],
|
||||||
|
stats: dict,
|
||||||
|
) -> None:
|
||||||
|
"""One kind of the overlay (movies, shows or episodes) — see `sync_overlay`."""
|
||||||
|
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
||||||
|
by_guid: dict[str, list[PlexItem | PlexShow]] = {}
|
||||||
|
for r in rows:
|
||||||
|
if r.guid: # the passes select guid-bearing rows only
|
||||||
|
by_guid.setdefault(r.guid, []).append(r)
|
||||||
|
# The HU listing is read first and applied after, so the clear can wait for its verdict.
|
||||||
|
hu_by_guid, complete = _read_hu_listing(plex, lib, item_type, by_guid)
|
||||||
|
episodes = item_type == plex_sync._TYPE_EPISODE
|
||||||
|
if complete:
|
||||||
|
_clear_hu(by_guid, episodes)
|
||||||
|
else:
|
||||||
|
stats["overlay_incomplete"] = stats.get("overlay_incomplete", 0) + 1
|
||||||
|
log.warning(
|
||||||
|
"the HU listing ended before the server's total; existing HU text is kept",
|
||||||
|
extra=logs.event("plex.sync.overlay_incomplete", library=lib.plex_key),
|
||||||
|
)
|
||||||
|
for g, meta in hu_by_guid.items():
|
||||||
|
targets = by_guid[g]
|
||||||
|
title_hu = (meta.get("title") or "").strip() or None
|
||||||
|
summary_hu = meta.get("summary")
|
||||||
|
tagline_hu = (meta.get("tagline") or "").strip() or None
|
||||||
|
genres_hu = hu_genres(plex_sync._tags(meta, "Genre"))
|
||||||
|
for row in targets:
|
||||||
|
row.title_hu = title_hu
|
||||||
|
row.summary_hu = summary_hu
|
||||||
|
if not episodes:
|
||||||
|
row.tagline_hu = tagline_hu
|
||||||
|
row.genres_hu = genres_hu or row.genres_hu
|
||||||
|
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
||||||
|
|
||||||
|
|
||||||
|
def _read_hu_listing(
|
||||||
|
plex: PlexClient, lib: PlexLibrary, item_type: int, by_guid: dict
|
||||||
|
) -> tuple[dict[str, dict], bool]:
|
||||||
|
"""The HU section's entries for the guids the EN side carries, and whether the walk reached the
|
||||||
|
server's total. The first HU entry for a guid wins; a dup HU item can't overwrite or double-count."""
|
||||||
|
progress: dict = {}
|
||||||
|
hu_by_guid: dict[str, dict] = {}
|
||||||
|
for meta in plex_sync._paginate(plex, lib.plex_key, item_type, progress):
|
||||||
|
g = plex_sync._canonical_guid(meta)
|
||||||
|
if g and g in by_guid and g not in hu_by_guid:
|
||||||
|
hu_by_guid[g] = meta
|
||||||
|
return hu_by_guid, bool(progress.get("complete"))
|
||||||
|
|
||||||
|
|
||||||
|
def _clear_hu(by_guid: dict, episodes: bool) -> None:
|
||||||
|
"""Reconcile step: clear HU fields on all guid-bearing rows; the matches are re-set after. Only
|
||||||
|
rows that actually held HU data become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is
|
||||||
|
cheap. The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking
|
||||||
|
one, from its own English list — otherwise the HU genre list mixed "Drama" (no twin) with "Dráma".
|
||||||
|
Only after a COMPLETE listing: one a Plex hiccup cut short would clear the HU text of every title
|
||||||
|
it failed to reach, until the next full sync (E32 S2, the watch reconcile's twin)."""
|
||||||
|
for group in by_guid.values():
|
||||||
|
for row in group:
|
||||||
|
row.title_hu = row.summary_hu = None
|
||||||
|
if not episodes:
|
||||||
|
row.tagline_hu = None
|
||||||
|
row.genres_hu = hu_genres(row.genres)
|
||||||
@@ -245,6 +245,9 @@ def _probe_video_kind(src: Path) -> str:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
|
# A non-zero exit is a failed probe (CalledProcessError, a SubprocessError → the branch
|
||||||
|
# below), not an answer: whatever it printed before refusing is not the file's truth.
|
||||||
|
check=True,
|
||||||
)
|
)
|
||||||
except (subprocess.SubprocessError, OSError) as exc:
|
except (subprocess.SubprocessError, OSError) as exc:
|
||||||
# The safe kind is a real answer, but it is the answer for an 8-bit SDR source — a blind
|
# The safe kind is a real answer, but it is the answer for an 8-bit SDR source — a blind
|
||||||
@@ -502,6 +505,9 @@ def _probe_audio_langs(src: Path) -> list[str]:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
|
# A non-zero exit is a failed probe (CalledProcessError, a SubprocessError → the branch
|
||||||
|
# below), not an answer: whatever it printed before refusing is not the file's truth.
|
||||||
|
check=True,
|
||||||
)
|
)
|
||||||
return [ln.strip() for ln in out.stdout.splitlines()]
|
return [ln.strip() for ln in out.stdout.splitlines()]
|
||||||
except (subprocess.SubprocessError, OSError) as exc:
|
except (subprocess.SubprocessError, OSError) as exc:
|
||||||
@@ -615,6 +621,9 @@ def _probe_first_pts(seg: Path) -> float | None:
|
|||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
timeout=10,
|
timeout=10,
|
||||||
|
# A non-zero exit is a failed probe (CalledProcessError, a SubprocessError → the branch
|
||||||
|
# below), not an answer: whatever it printed before refusing is not the file's truth.
|
||||||
|
check=True,
|
||||||
)
|
)
|
||||||
for line in out.stdout.splitlines():
|
for line in out.stdout.splitlines():
|
||||||
tok = line.strip().strip(",")
|
tok = line.strip().strip(",")
|
||||||
@@ -736,8 +745,7 @@ def start_session(
|
|||||||
# check-and-replace happens under ONE _lock, so a newer same-key session (a concurrent
|
# check-and-replace happens under ONE _lock, so a newer same-key session (a concurrent
|
||||||
# seek-restart) installed between the fallback's decision and here is never clobbered. When the
|
# seek-restart) installed between the fallback's decision and here is never clobbered. When the
|
||||||
# registered session is no longer `replace_only`, we abort and return None (the caller yields to
|
# registered session is no longer `replace_only`, we abort and return None (the caller yields to
|
||||||
# whoever is registered now). The INITIAL spawn passes replace_only=None and always supersedes —
|
# whoever is registered now). The INITIAL spawn never gets None — see its call site.
|
||||||
# that is the seek-restart's intended last-writer-wins.
|
|
||||||
# Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and
|
# Software-fallback encode tuning — read only on the SW transcode path (the VAAPI path and
|
||||||
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
# remux ignore preset/crf/max_height, and remux must not touch the DB for them).
|
||||||
preset, crf, max_height = "veryfast", 20, 0
|
preset, crf, max_height = "veryfast", 20, 0
|
||||||
@@ -814,6 +822,7 @@ def start_session(
|
|||||||
return s
|
return s
|
||||||
|
|
||||||
s = _spawn(*_select_encode(db, item, transcode, src))
|
s = _spawn(*_select_encode(db, item, transcode, src))
|
||||||
|
assert s is not None # replace_only=None always supersedes: a seek-restart's last-writer-wins
|
||||||
# Wait for the first video segment outside the lock (this blocks on ffmpeg, and we must not hold up
|
# Wait for the first video segment outside the lock (this blocks on ffmpeg, and we must not hold up
|
||||||
# other sessions). Pass the proc so a HW session that dies at VAAPI init fails fast (not after 25s).
|
# other sessions). Pass the proc so a HW session that dies at VAAPI init fails fast (not after 25s).
|
||||||
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
ready = wait_for(_seg(0), timeout=25.0, proc=s.proc)
|
||||||
|
|||||||
@@ -162,7 +162,8 @@ def ffmpeg_to_vtt(src: str, sub_ord: int | None) -> str:
|
|||||||
"webvtt",
|
"webvtt",
|
||||||
"pipe:1",
|
"pipe:1",
|
||||||
]
|
]
|
||||||
r = subprocess.run(cmd, capture_output=True, timeout=EXTRACT_TIMEOUT_S)
|
# check=False: the exit status is read on the next line, with ffmpeg's own stderr as the reason.
|
||||||
|
r = subprocess.run(cmd, capture_output=True, timeout=EXTRACT_TIMEOUT_S, check=False)
|
||||||
if r.returncode != 0 or not r.stdout:
|
if r.returncode != 0 or not r.stdout:
|
||||||
raise PlexError(
|
raise PlexError(
|
||||||
(r.stderr or b"").decode("utf-8", "replace")[:200] or "ffmpeg subtitle convert failed"
|
(r.stderr or b"").decode("utf-8", "replace")[:200] or "ffmpeg subtitle convert failed"
|
||||||
|
|||||||
+58
-123
@@ -15,7 +15,6 @@ dead `enabled` column) is a separate follow-up. Cast + intro/credit markers are
|
|||||||
item-detail/player path (P2), not here.
|
item-detail/player path (P2), not here.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from collections.abc import Iterable
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
@@ -23,7 +22,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app import logs, sysconfig
|
from app import logs, sysconfig
|
||||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||||
from app.plex import detail, people
|
from app.plex import detail, overlay, people
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
from app.plex.genres import hu_genres
|
from app.plex.genres import hu_genres
|
||||||
from app.plex.lookups import by_rating_key
|
from app.plex.lookups import by_rating_key
|
||||||
@@ -194,20 +193,6 @@ def _enabled_section_keys(db: Session) -> set[str] | None:
|
|||||||
return keys or None
|
return keys or None
|
||||||
|
|
||||||
|
|
||||||
def _overlay_map(db: Session) -> dict[str, str]:
|
|
||||||
"""The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
|
|
||||||
('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
|
|
||||||
Malformed pairs are skipped."""
|
|
||||||
raw = sysconfig.get_str(db, "plex_overlay_map")
|
|
||||||
out: dict[str, str] = {}
|
|
||||||
for pair in raw.split(","):
|
|
||||||
hu, sep, en = pair.partition(":")
|
|
||||||
hu, en = hu.strip(), en.strip()
|
|
||||||
if sep and hu and en:
|
|
||||||
out[hu] = en
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def sync(db: Session) -> dict:
|
def sync(db: Session) -> dict:
|
||||||
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
|
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
|
||||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||||
@@ -223,7 +208,7 @@ def sync(db: Session) -> dict:
|
|||||||
"pruned": 0,
|
"pruned": 0,
|
||||||
}
|
}
|
||||||
wanted = _enabled_section_keys(db)
|
wanted = _enabled_section_keys(db)
|
||||||
overlay_map = _overlay_map(db) # {huKey: enKey}
|
overlay_map = overlay.overlay_map(db) # {huKey: enKey}
|
||||||
overlay_keys = set(overlay_map)
|
overlay_keys = set(overlay_map)
|
||||||
try:
|
try:
|
||||||
with PlexClient(db) as plex:
|
with PlexClient(db) as plex:
|
||||||
@@ -267,13 +252,13 @@ def sync(db: Session) -> dict:
|
|||||||
lib.metadata_lang = "hu"
|
lib.metadata_lang = "hu"
|
||||||
lib.overlay_of = en_lib.id
|
lib.overlay_of = en_lib.id
|
||||||
db.flush()
|
db.flush()
|
||||||
_sync_overlay(db, plex, lib, en_lib.id, stats)
|
overlay.sync_overlay(db, plex, lib, en_lib.id, stats)
|
||||||
db.commit()
|
db.commit()
|
||||||
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
|
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
|
||||||
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
||||||
try:
|
try:
|
||||||
stats["people_enriched"] = _enrich_people_batch(db, plex)
|
stats["people_enriched"] = _enrich_people_batch(db, plex)
|
||||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
except Exception: # best-effort; never fail the sync over a backfill
|
||||||
# With the traceback: a blind catch over a batch that fetches AND writes can be
|
# With the traceback: a blind catch over a batch that fetches AND writes can be
|
||||||
# a Plex shape change, a DB error or our own bug, and `%s` told them apart badly.
|
# a Plex shape change, a DB error or our own bug, and `%s` told them apart badly.
|
||||||
log.exception("the Plex people-enrich batch failed")
|
log.exception("the Plex people-enrich batch failed")
|
||||||
@@ -282,7 +267,7 @@ def sync(db: Session) -> dict:
|
|||||||
# successive runs. Only HW-decodable-codec transcodes are probed — the rest are SW by codec.
|
# successive runs. Only HW-decodable-codec transcodes are probed — the rest are SW by codec.
|
||||||
try:
|
try:
|
||||||
stats["media_probed"] = _probe_media_batch(db, plex)
|
stats["media_probed"] = _probe_media_batch(db, plex)
|
||||||
except Exception: # noqa: BLE001 — best-effort; never fail the sync over a backfill
|
except Exception: # best-effort; never fail the sync over a backfill
|
||||||
log.exception("the Plex media-probe batch failed")
|
log.exception("the Plex media-probe batch failed")
|
||||||
# silent-ok: the return value IS the report. `{"skipped": …}` goes to the scheduler, which
|
# silent-ok: the return value IS the report. `{"skipped": …}` goes to the scheduler, which
|
||||||
# logs the job result, and to the admin route, which answers it to the caller.
|
# logs the job result, and to the admin route, which answers it to the caller.
|
||||||
@@ -296,7 +281,7 @@ def sync(db: Session) -> dict:
|
|||||||
# appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata.
|
# appear in ANY item's cast/directors — drop those rows, mirroring how Plex drops orphaned metadata.
|
||||||
try:
|
try:
|
||||||
stats["people_pruned"] = people.prune_orphan_people(db)
|
stats["people_pruned"] = people.prune_orphan_people(db)
|
||||||
except Exception: # noqa: BLE001 — best-effort housekeeping; never fail the sync over it
|
except Exception: # best-effort housekeeping; never fail the sync over it
|
||||||
log.exception("the Plex people prune failed")
|
log.exception("the Plex people prune failed")
|
||||||
log.info("Plex sync done: %s", stats)
|
log.info("Plex sync done: %s", stats)
|
||||||
return stats
|
return stats
|
||||||
@@ -433,7 +418,10 @@ def _paginate(plex: PlexClient, key: str, item_type: int, progress: dict | None
|
|||||||
if progress is not None:
|
if progress is not None:
|
||||||
# A truthy `total` we reached = a trustworthy complete listing. `total` unknown/zero-with-rows
|
# A truthy `total` we reached = a trustworthy complete listing. `total` unknown/zero-with-rows
|
||||||
# (a Plex that doesn't report totalSize) is treated as NOT complete, so prune stays conservative.
|
# (a Plex that doesn't report totalSize) is treated as NOT complete, so prune stays conservative.
|
||||||
|
# `total` itself is kept for a caller to whom a server that SAID 0 is news it can trust (the
|
||||||
|
# watch reconcile); an empty mirror or overlay must not read it so (E32 S2).
|
||||||
progress["complete"] = bool(total) and start >= total
|
progress["complete"] = bool(total) and start >= total
|
||||||
|
progress["total"] = total
|
||||||
|
|
||||||
|
|
||||||
# The section listing exposes at most the top ~20 cast (Plex's own cap); a stored cast longer than this
|
# The section listing exposes at most the top ~20 cast (Plex's own cap); a stored cast longer than this
|
||||||
@@ -661,98 +649,6 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
|||||||
_prune_stale(db, shows, shows_seen, stats, complete=shows_progress.get("complete", False))
|
_prune_stale(db, shows, shows_seen, stats, complete=shows_progress.get("complete", False))
|
||||||
|
|
||||||
|
|
||||||
def _sync_overlay(
|
|
||||||
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
|
||||||
) -> None:
|
|
||||||
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
|
||||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/tagline/genres
|
|
||||||
differ by language; the language-neutral people_text feeds both search vectors. A show section's
|
|
||||||
episodes are overlaid too (E30): the HU section lists every episode with its Hungarian title and
|
|
||||||
summary, and the episode rows live in the EN show library.
|
|
||||||
|
|
||||||
This is a full RECONCILE: every guid-bearing EN row's HU fields are cleared first, then the HU
|
|
||||||
section repopulates the matches — so an EN row whose HU twin vanished (or was re-guid'd) falls back
|
|
||||||
to EN instead of keeping stale Hungarian metadata. HU items with no EN match are skipped."""
|
|
||||||
if lib.kind == "movie":
|
|
||||||
passes = [
|
|
||||||
(
|
|
||||||
_TYPE_MOVIE,
|
|
||||||
select(PlexItem).where(
|
|
||||||
PlexItem.library_id == en_lib_id,
|
|
||||||
PlexItem.kind == "movie",
|
|
||||||
PlexItem.guid.is_not(None),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
]
|
|
||||||
else:
|
|
||||||
passes = [
|
|
||||||
(
|
|
||||||
_TYPE_SHOW,
|
|
||||||
select(PlexShow).where(
|
|
||||||
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
|
|
||||||
),
|
|
||||||
),
|
|
||||||
(
|
|
||||||
_TYPE_EPISODE,
|
|
||||||
select(PlexItem).where(
|
|
||||||
PlexItem.library_id == en_lib_id,
|
|
||||||
PlexItem.kind == "episode",
|
|
||||||
PlexItem.guid.is_not(None),
|
|
||||||
),
|
|
||||||
),
|
|
||||||
]
|
|
||||||
for item_type, rows_q in passes:
|
|
||||||
_overlay_pass(db, plex, lib, item_type, db.execute(rows_q).scalars(), stats)
|
|
||||||
db.flush()
|
|
||||||
|
|
||||||
|
|
||||||
def _overlay_pass(
|
|
||||||
db: Session,
|
|
||||||
plex: PlexClient,
|
|
||||||
lib: PlexLibrary,
|
|
||||||
item_type: int,
|
|
||||||
rows: Iterable[PlexItem] | Iterable[PlexShow],
|
|
||||||
stats: dict,
|
|
||||||
) -> None:
|
|
||||||
"""One kind of the overlay (movies, shows or episodes) — see `_sync_overlay`."""
|
|
||||||
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
|
||||||
by_guid: dict[str, list[PlexItem | PlexShow]] = {}
|
|
||||||
for r in rows:
|
|
||||||
if r.guid: # the passes select guid-bearing rows only
|
|
||||||
by_guid.setdefault(r.guid, []).append(r)
|
|
||||||
# Reconcile step: clear HU fields on all guid-bearing rows. Only rows that actually held HU data
|
|
||||||
# become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is cheap; matches are re-set below.
|
|
||||||
# The genres are the exception: every row gets Hungarian ones, from its HU twin or, lacking one, from
|
|
||||||
# its own English list — otherwise the HU genre list mixed "Drama" (no twin) with "Dráma".
|
|
||||||
episodes = item_type == _TYPE_EPISODE
|
|
||||||
for group in by_guid.values():
|
|
||||||
for row in group:
|
|
||||||
row.title_hu = row.summary_hu = None
|
|
||||||
if not episodes:
|
|
||||||
row.tagline_hu = None
|
|
||||||
row.genres_hu = hu_genres(row.genres)
|
|
||||||
seen: set[str] = (
|
|
||||||
set()
|
|
||||||
) # first HU entry for a guid wins; a dup HU item can't overwrite or double-count
|
|
||||||
for meta in _paginate(plex, lib.plex_key, item_type):
|
|
||||||
g = _canonical_guid(meta)
|
|
||||||
targets = by_guid.get(g) if g else None
|
|
||||||
if not targets or g in seen:
|
|
||||||
continue
|
|
||||||
seen.add(g)
|
|
||||||
title_hu = (meta.get("title") or "").strip() or None
|
|
||||||
summary_hu = meta.get("summary")
|
|
||||||
tagline_hu = (meta.get("tagline") or "").strip() or None
|
|
||||||
genres_hu = hu_genres(_tags(meta, "Genre"))
|
|
||||||
for row in targets:
|
|
||||||
row.title_hu = title_hu
|
|
||||||
row.summary_hu = summary_hu
|
|
||||||
if not episodes:
|
|
||||||
row.tagline_hu = tagline_hu
|
|
||||||
row.genres_hu = genres_hu or row.genres_hu
|
|
||||||
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
|
||||||
|
|
||||||
|
|
||||||
def _upsert_collection(
|
def _upsert_collection(
|
||||||
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
|
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
|
||||||
) -> PlexCollection:
|
) -> PlexCollection:
|
||||||
@@ -778,9 +674,11 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
|||||||
meta = plex.metadata(rk) or {}
|
meta = plex.metadata(rk) or {}
|
||||||
col = _upsert_collection(db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta)
|
col = _upsert_collection(db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta)
|
||||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||||
new_members = {
|
children = _believed_children(plex, rk, meta)
|
||||||
str(c.get("ratingKey")) for c in plex.collection_children(rk) if c.get("ratingKey")
|
if children is None:
|
||||||
}
|
db.flush()
|
||||||
|
return col
|
||||||
|
new_members = {str(c.get("ratingKey")) for c in children if c.get("ratingKey")}
|
||||||
current = db.execute(select(Model).where(Model.collection_keys.contains([rk]))).scalars().all()
|
current = db.execute(select(Model).where(Model.collection_keys.contains([rk]))).scalars().all()
|
||||||
for row in current: # drop rk from rows that are no longer members
|
for row in current: # drop rk from rows that are no longer members
|
||||||
if row.rating_key not in new_members:
|
if row.rating_key not in new_members:
|
||||||
@@ -793,6 +691,22 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
|||||||
return col
|
return col
|
||||||
|
|
||||||
|
|
||||||
|
def _believed_children(plex: PlexClient, rk: str, meta: dict) -> list[dict] | None:
|
||||||
|
"""A collection's members, or None when the answer is not believed: no members from a collection
|
||||||
|
whose own listing counts some (`childCount`) is a glitch, not a collection emptied — the same
|
||||||
|
empty-answer rule as the collection listing's (E32 review). Rebuilding on it showed the
|
||||||
|
collection empty until the next sync."""
|
||||||
|
children = plex.collection_children(rk)
|
||||||
|
count = str(meta.get("childCount") or "0")
|
||||||
|
if children or not count.isdigit() or int(count) == 0:
|
||||||
|
return children
|
||||||
|
log.warning(
|
||||||
|
"Plex listed no members for a non-empty collection; keeping its membership",
|
||||||
|
extra=logs.event("plex.sync.collection_children_empty", collection=rk),
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
||||||
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
|
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
|
||||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||||
@@ -804,6 +718,19 @@ def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
|||||||
db.flush()
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def _rebuild_membership(
|
||||||
|
db: Session, lib: PlexLibrary, membership: dict[str, list[str]], kept: set[str]
|
||||||
|
) -> None:
|
||||||
|
"""Rebuild collection_keys on every member row of this library (clear + set), carrying over the
|
||||||
|
keys of a collection whose empty answer was not believed (`_believed_children`)."""
|
||||||
|
for Model in (PlexItem, PlexShow):
|
||||||
|
for row in db.execute(select(Model).where(Model.library_id == lib.id)).scalars():
|
||||||
|
held = [k for k in (row.collection_keys or []) if k in kept]
|
||||||
|
keys = [*membership.get(row.rating_key, []), *held]
|
||||||
|
row.collection_keys = list(dict.fromkeys(keys)) or None
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
||||||
"""Mirror a library's collections + membership. Reads never touch Plex afterwards. Membership is
|
"""Mirror a library's collections + membership. Reads never touch Plex afterwards. Membership is
|
||||||
written as `collection_keys` on member movies (plex_items) / shows (plex_shows). A full reconcile:
|
written as `collection_keys` on member movies (plex_items) / shows (plex_shows). A full reconcile:
|
||||||
@@ -815,7 +742,17 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
|||||||
select(PlexCollection).where(PlexCollection.library_id == lib.id)
|
select(PlexCollection).where(PlexCollection.library_id == lib.id)
|
||||||
).scalars()
|
).scalars()
|
||||||
}
|
}
|
||||||
|
if not cols and existing:
|
||||||
|
# The catalog's own rule (see the module doc): an empty listing while we hold rows is a Plex
|
||||||
|
# glitch, not a library emptied of collections. Reconciling on it dropped every collection
|
||||||
|
# (its user-set `editable` flag with it) and every member's keys (E32 S2).
|
||||||
|
log.warning(
|
||||||
|
"Plex listed no collections for a library that has some; keeping them this run",
|
||||||
|
extra=logs.event("plex.sync.collections_empty", library=lib.plex_key),
|
||||||
|
)
|
||||||
|
return
|
||||||
seen: set[str] = set()
|
seen: set[str] = set()
|
||||||
|
kept: set[str] = set() # collections whose empty children answer is not believed
|
||||||
membership: dict[str, list[str]] = {} # member rating_key -> [collection rating_key, ...]
|
membership: dict[str, list[str]] = {} # member rating_key -> [collection rating_key, ...]
|
||||||
for meta in cols:
|
for meta in cols:
|
||||||
rk = str(meta.get("ratingKey") or "")
|
rk = str(meta.get("ratingKey") or "")
|
||||||
@@ -823,7 +760,10 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
|||||||
continue
|
continue
|
||||||
_upsert_collection(db, existing.get(rk), rk, lib.id, meta)
|
_upsert_collection(db, existing.get(rk), rk, lib.id, meta)
|
||||||
seen.add(rk)
|
seen.add(rk)
|
||||||
for child in plex.collection_children(rk):
|
children = _believed_children(plex, rk, meta)
|
||||||
|
if children is None:
|
||||||
|
kept.add(rk)
|
||||||
|
for child in children or []:
|
||||||
crk = str(child.get("ratingKey") or "")
|
crk = str(child.get("ratingKey") or "")
|
||||||
if crk:
|
if crk:
|
||||||
membership.setdefault(crk, []).append(rk)
|
membership.setdefault(crk, []).append(rk)
|
||||||
@@ -832,9 +772,4 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
|||||||
for rk, col in existing.items():
|
for rk, col in existing.items():
|
||||||
if rk not in seen:
|
if rk not in seen:
|
||||||
db.delete(col)
|
db.delete(col)
|
||||||
# Rebuild collection_keys on every member row of this library (clear + set).
|
_rebuild_membership(db, lib, membership, kept)
|
||||||
for row in db.execute(select(PlexItem).where(PlexItem.library_id == lib.id)).scalars():
|
|
||||||
row.collection_keys = membership.get(row.rating_key) or None
|
|
||||||
for row in db.execute(select(PlexShow).where(PlexShow.library_id == lib.id)).scalars():
|
|
||||||
row.collection_keys = membership.get(row.rating_key) or None
|
|
||||||
db.flush()
|
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app import logs, sysconfig
|
from app import logs, sysconfig
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.models import PlexItem, PlexLink, PlexState
|
from app.models import PlexItem, PlexLibrary, PlexLink, PlexState
|
||||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
from app.plex.sync import (
|
from app.plex.sync import (
|
||||||
_TYPE_EPISODE,
|
_TYPE_EPISODE,
|
||||||
@@ -72,13 +72,27 @@ def _rk_to_id(db: Session) -> dict[str, int]:
|
|||||||
|
|
||||||
def _scan_plex_states(
|
def _scan_plex_states(
|
||||||
db: Session, plex: PlexClient
|
db: Session, plex: PlexClient
|
||||||
) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int]:
|
) -> tuple[dict[int, tuple[str, int, datetime | None, datetime | None]], int, set[str], set[str]]:
|
||||||
"""Scan the enabled movie/show sections and return ``{item_id: (status, pos, watched_at,
|
"""Scan the enabled movie/show sections and return ``{item_id: (status, pos, watched_at,
|
||||||
prog_at)}`` for every mirrored leaf Plex holds a watch signal for, plus the scanned-leaf count.
|
prog_at)}`` for every mirrored leaf Plex holds a watch signal for, the scanned-leaf count, the
|
||||||
Shared by the one-time import and the full reconcile (both need Plex's whole current picture)."""
|
keys of the sections SETTLED — walked to the server's total (or said empty by a server we mirror
|
||||||
|
nothing from) — and the keys of every section walked.
|
||||||
|
Shared by the one-time import and the full reconcile (both need Plex's whole current picture).
|
||||||
|
The settled keys matter to the reconcile alone: it reads "absent from the scan" as "Plex
|
||||||
|
un-watched it", which a walk cut short by a Plex hiccup does not mean (see `_paginate`). Settled
|
||||||
|
is decided PER SECTION: one section that never settles (a library emptied on Plex while its
|
||||||
|
mirrored items are kept) must not stop un-watches syncing in all the others (E32 review)."""
|
||||||
item_id_by_rk = _rk_to_id(db)
|
item_id_by_rk = _rk_to_id(db)
|
||||||
wanted = _enabled_section_keys(db)
|
wanted = _enabled_section_keys(db)
|
||||||
|
# The sections we mirror something from: a total of 0 from one of those contradicts the mirror.
|
||||||
|
mirrored = set(
|
||||||
|
db.execute(
|
||||||
|
select(PlexLibrary.plex_key).join(PlexItem, PlexItem.library_id == PlexLibrary.id)
|
||||||
|
).scalars()
|
||||||
|
)
|
||||||
scanned = 0
|
scanned = 0
|
||||||
|
settled: set[str] = set()
|
||||||
|
walked: set[str] = set()
|
||||||
out: dict[int, tuple[str, int, datetime | None, datetime | None]] = {}
|
out: dict[int, tuple[str, int, datetime | None, datetime | None]] = {}
|
||||||
for s in plex.sections():
|
for s in plex.sections():
|
||||||
stype = s.get("type")
|
stype = s.get("type")
|
||||||
@@ -88,7 +102,10 @@ def _scan_plex_states(
|
|||||||
if wanted is not None and key not in wanted:
|
if wanted is not None and key not in wanted:
|
||||||
continue
|
continue
|
||||||
item_type = _TYPE_MOVIE if stype == "movie" else _TYPE_EPISODE
|
item_type = _TYPE_MOVIE if stype == "movie" else _TYPE_EPISODE
|
||||||
for meta in _paginate(plex, key, item_type):
|
walked.add(key)
|
||||||
|
progress: dict = {}
|
||||||
|
before = scanned
|
||||||
|
for meta in _paginate(plex, key, item_type, progress):
|
||||||
scanned += 1
|
scanned += 1
|
||||||
item_id = item_id_by_rk.get(str(meta.get("ratingKey") or ""))
|
item_id = item_id_by_rk.get(str(meta.get("ratingKey") or ""))
|
||||||
if item_id is None:
|
if item_id is None:
|
||||||
@@ -98,7 +115,15 @@ def _scan_plex_states(
|
|||||||
)
|
)
|
||||||
if res is not None:
|
if res is not None:
|
||||||
out[item_id] = res
|
out[item_id] = res
|
||||||
return out, scanned
|
# A section the server SAYS is empty (total 0, nothing listed) settles too. Believed only for
|
||||||
|
# a section we mirror nothing from: one whose items we hold answering 0 is a Plex that lost
|
||||||
|
# the library for a moment, not one that un-watched all of it (E32 review) — so it is left
|
||||||
|
# unsettled, and its rows are not cleared.
|
||||||
|
listed = scanned > before
|
||||||
|
said_empty = progress.get("total") == 0 and not listed and key not in mirrored
|
||||||
|
if (progress.get("complete") and listed) or said_empty:
|
||||||
|
settled.add(key)
|
||||||
|
return out, scanned, settled, walked
|
||||||
|
|
||||||
|
|
||||||
def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
||||||
@@ -111,7 +136,8 @@ def import_owner_watch_state(db: Session, link: PlexLink) -> dict:
|
|||||||
stats = {"watched": 0, "in_progress": 0, "scanned": 0}
|
stats = {"watched": 0, "in_progress": 0, "scanned": 0}
|
||||||
try:
|
try:
|
||||||
with PlexClient(db) as plex:
|
with PlexClient(db) as plex:
|
||||||
scanned_states, stats["scanned"] = _scan_plex_states(db, plex)
|
# A partial scan is still safe here: the import only ever ADDS what Plex reports.
|
||||||
|
scanned_states, stats["scanned"], _settled, _walked = _scan_plex_states(db, plex)
|
||||||
# One row per item (keyed by item_id → inherently de-duped) then bulk-UPSERT. An upsert is
|
# One row per item (keyed by item_id → inherently de-duped) then bulk-UPSERT. An upsert is
|
||||||
# idempotent AND concurrency-safe: a repeated/overlapping enable (the toggle firing twice
|
# idempotent AND concurrency-safe: a repeated/overlapping enable (the toggle firing twice
|
||||||
# while the multi-second import runs) can't raise a duplicate-key error — it re-updates.
|
# while the multi-second import runs) can't raise a duplicate-key error — it re-updates.
|
||||||
@@ -590,6 +616,28 @@ def run_plex_watch_sync(db: Session) -> dict:
|
|||||||
return total
|
return total
|
||||||
|
|
||||||
|
|
||||||
|
def _clear_unwatched(db: Session, user_id: int, plex_states: dict, settled: set[str]) -> int:
|
||||||
|
"""Clear the user's rows Plex no longer has a record for — the mirrored ones (Plex un-watched
|
||||||
|
them); hidden and dirty rows stay (a dirty one is re-pushed by `_repush_dirty`). Only rows of the
|
||||||
|
sections the scan SETTLED: a walk a Plex hiccup cut short leaves out rows it merely failed to
|
||||||
|
reach, and clearing those un-watched them here until the next complete run (E32 S2); an item of
|
||||||
|
a library the allowlist leaves out is absent from every scan, and was cleared on every run (E32
|
||||||
|
review)."""
|
||||||
|
n = 0
|
||||||
|
rows = db.execute(
|
||||||
|
select(PlexState)
|
||||||
|
.join(PlexItem, PlexItem.id == PlexState.item_id)
|
||||||
|
.join(PlexLibrary, PlexLibrary.id == PlexItem.library_id)
|
||||||
|
.where(PlexState.user_id == user_id, PlexLibrary.plex_key.in_(settled))
|
||||||
|
)
|
||||||
|
for st in rows.scalars().all():
|
||||||
|
if st.item_id in plex_states or st.status == "hidden" or not st.synced_to_plex:
|
||||||
|
continue
|
||||||
|
db.delete(st)
|
||||||
|
n += 1
|
||||||
|
return n
|
||||||
|
|
||||||
|
|
||||||
def run_plex_watch_reconcile(db: Session) -> dict:
|
def run_plex_watch_reconcile(db: Session) -> dict:
|
||||||
"""Scheduler job (default daily) — the authoritative full reconcile. Rescans every section and,
|
"""Scheduler job (default daily) — the authoritative full reconcile. Rescans every section and,
|
||||||
per item, uses `synced_to_plex` to settle what the incremental feed can't:
|
per item, uses `synced_to_plex` to settle what the incremental feed can't:
|
||||||
@@ -608,7 +656,7 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
|||||||
try:
|
try:
|
||||||
with PlexClient(db) as plex:
|
with PlexClient(db) as plex:
|
||||||
_resolve_account_id(db, plex, link)
|
_resolve_account_id(db, plex, link)
|
||||||
plex_states, scanned = _scan_plex_states(db, plex)
|
plex_states, scanned, settled, walked = _scan_plex_states(db, plex)
|
||||||
total["scanned"] += scanned
|
total["scanned"] += scanned
|
||||||
|
|
||||||
for item_id, (status, pos, watched_at, prog_at) in plex_states.items():
|
for item_id, (status, pos, watched_at, prog_at) in plex_states.items():
|
||||||
@@ -616,18 +664,19 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
|||||||
if ch:
|
if ch:
|
||||||
total["applied"] += 1
|
total["applied"] += 1
|
||||||
|
|
||||||
# Siftlode rows Plex no longer has a record for: clear the ones that were mirrored
|
total["cleared"] += _clear_unwatched(db, link.user_id, plex_states, settled)
|
||||||
# (Plex un-watched them); leave hidden + dirty (dirty gets re-pushed just below).
|
if unsettled := sorted(walked - settled):
|
||||||
for st in (
|
total["incomplete"] = total.get("incomplete", 0) + 1
|
||||||
db.execute(select(PlexState).where(PlexState.user_id == link.user_id))
|
log.warning(
|
||||||
.scalars()
|
"Plex sections %s were cut short or answered empty; their rows are not cleared"
|
||||||
.all()
|
" this run",
|
||||||
):
|
", ".join(unsettled),
|
||||||
if st.item_id in plex_states or st.status == "hidden":
|
extra=logs.event(
|
||||||
continue
|
"plex.watch.reconcile_incomplete",
|
||||||
if st.synced_to_plex:
|
user_id=link.user_id,
|
||||||
db.delete(st)
|
sections=unsettled,
|
||||||
total["cleared"] += 1
|
),
|
||||||
|
)
|
||||||
|
|
||||||
total["repushed"] += _repush_dirty(db, plex, link)
|
total["repushed"] += _repush_dirty(db, plex, link)
|
||||||
link.last_watch_sync_at = datetime.now(timezone.utc)
|
link.last_watch_sync_at = datetime.now(timezone.utc)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ its own thread/context) never cross wires. When no sink is bound (e.g. a manual
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import contextvars
|
import contextvars
|
||||||
from typing import Callable, Optional
|
from typing import Callable, Literal, Optional
|
||||||
|
|
||||||
Sink = Callable[[int, Optional[int], Optional[str]], None]
|
Sink = Callable[[int, Optional[int], Optional[str]], None]
|
||||||
|
|
||||||
@@ -48,7 +48,9 @@ class bind:
|
|||||||
self._token = _sink.set(self._sink)
|
self._token = _sink.set(self._sink)
|
||||||
return self
|
return self
|
||||||
|
|
||||||
def __exit__(self, *exc) -> bool:
|
# Literal[False], not bool: this never swallows the job's exception, and saying so is what lets
|
||||||
|
# a type checker know a name assigned inside `with bind(...)` is bound after it.
|
||||||
|
def __exit__(self, *exc) -> Literal[False]:
|
||||||
if self._token is not None:
|
if self._token is not None:
|
||||||
_sink.reset(self._token)
|
_sink.reset(self._token)
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -60,14 +60,17 @@ def owned_or_404(
|
|||||||
return obj
|
return obj
|
||||||
|
|
||||||
|
|
||||||
def youtube_refusal(exc: Exception) -> HTTPException:
|
def youtube_refusal(exc: Exception, *, playlist: bool = False) -> HTTPException:
|
||||||
"""The answer to an action YouTube refused: a 422 whose reason the UI translates
|
"""The answer to an action YouTube refused: a 422 whose reason the UI translates
|
||||||
(`errors.youtube.<reason>`). One definition, because the routes had grown three shapes of it —
|
(`errors.youtube.<reason>`). One definition, because the routes had grown three shapes of it —
|
||||||
a bare English string quoting the upstream message, a playlist-scoped reason, and nothing at
|
a bare English string quoting the upstream message, a playlist-scoped reason, and nothing at
|
||||||
all (a raw 500 from the manual syncs). The upstream failure is already logged where it
|
all (a raw 500 from the manual syncs). The upstream failure is already logged where it
|
||||||
happened (`youtube.api.error`). A failure that is not the Data API's (the search scraper's) is
|
happened (`youtube.api.error`). A failure that is not the Data API's (the search scraper's) is
|
||||||
YouTube being unavailable to us — the retryable answer."""
|
YouTube being unavailable to us — the retryable answer. `playlist`: the route acts on one
|
||||||
reason = refusal_reason(exc) if isinstance(exc, YouTubeError) else "unavailable"
|
playlist the user picked, so a missing playlist is THAT one, gone (`refusal_reason`)."""
|
||||||
|
reason = (
|
||||||
|
refusal_reason(exc, playlist=playlist) if isinstance(exc, YouTubeError) else "unavailable"
|
||||||
|
)
|
||||||
return HTTPException(status_code=422, detail={"code": "youtube", "reason": reason})
|
return HTTPException(status_code=422, detail={"code": "youtube", "reason": reason})
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -436,7 +436,7 @@ def explore_channel(
|
|||||||
channel = db.get(Channel, channel_id)
|
channel = db.get(Channel, channel_id)
|
||||||
result = explore_ingest_page(db, yt, channel, page_token)
|
result = explore_ingest_page(db, yt, channel, page_token)
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"channel_id": channel_id,
|
"channel_id": channel_id,
|
||||||
@@ -597,7 +597,7 @@ def subscribe(
|
|||||||
if "subscriptionDuplicate" in exc.reason.split(","):
|
if "subscriptionDuplicate" in exc.reason.split(","):
|
||||||
yt_sub_id = ""
|
yt_sub_id = ""
|
||||||
else:
|
else:
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
|
# Enrich a stub channel (title/thumbnail/subscriber count) so it shows up properly right
|
||||||
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
|
# away, on BOTH the normal and the desync path — a resurfaced stub still needs metadata to
|
||||||
# render. Keep the client open (we're still inside the `with`); a metadata failure here
|
# render. Keep the client open (we're still inside the `with`); a metadata failure here
|
||||||
@@ -658,7 +658,7 @@ def unsubscribe(
|
|||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
|
# 422 (a real, explainable error → speaking modal), not 502 which the client treats as
|
||||||
# a transient "connection lost" gateway blip.
|
# a transient "connection lost" gateway blip.
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
db.delete(sub)
|
db.delete(sub)
|
||||||
db.commit()
|
db.commit()
|
||||||
return {"ok": True, "unsubscribed": channel_id}
|
return {"ok": True, "unsubscribed": channel_id}
|
||||||
|
|||||||
@@ -170,7 +170,7 @@ def add_pin(
|
|||||||
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
# between `resolve_ref` confirming it and this INSERT (a Plex sync dropping a removed
|
||||||
# item), which trips the FK. Semantically that is the same "no such thing" the pre-check
|
# item), which trips the FK. Semantically that is the same "no such thing" the pre-check
|
||||||
# returns above — re-raising handed the user a 500 they could do nothing with.
|
# returns above — re-raising handed the user a 500 they could do nothing with.
|
||||||
raise not_found("item")
|
raise not_found("item") from None
|
||||||
return {"ok": True, "id": winner.id, "created": False}
|
return {"ok": True, "id": winner.id, "created": False}
|
||||||
return {"ok": True, "id": pin.id, "created": True}
|
return {"ok": True, "id": pin.id, "created": True}
|
||||||
|
|
||||||
|
|||||||
@@ -322,7 +322,7 @@ def set_credential(
|
|||||||
try:
|
try:
|
||||||
cleaned = dlcreds.validate_cookies(body.cookies)
|
cleaned = dlcreds.validate_cookies(body.cookies)
|
||||||
except dlcreds.InvalidCookies as e:
|
except dlcreds.InvalidCookies as e:
|
||||||
raise HTTPException(status_code=400, detail={"code": "cookies", "reason": str(e)})
|
raise HTTPException(status_code=400, detail={"code": "cookies", "reason": str(e)}) from e
|
||||||
dlcreds.upsert_credential(db, user.id, host_key, cleaned)
|
dlcreds.upsert_credential(db, user.id, host_key, cleaned)
|
||||||
audit.record(
|
audit.record(
|
||||||
db,
|
db,
|
||||||
@@ -407,7 +407,7 @@ def enqueue_download(
|
|||||||
try:
|
try:
|
||||||
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
|
job = service.enqueue(db, user.id, source_kind, source_ref, spec, profile_id, display_name)
|
||||||
except quota.QuotaExceeded as e:
|
except quota.QuotaExceeded as e:
|
||||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
|
||||||
return _serialize_job(db, job)
|
return _serialize_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
@@ -520,7 +520,7 @@ def upload_download(
|
|||||||
display_name=upload.display_name,
|
display_name=upload.display_name,
|
||||||
)
|
)
|
||||||
except quota.QuotaExceeded as e:
|
except quota.QuotaExceeded as e:
|
||||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
|
||||||
return _serialize_job(db, job)
|
return _serialize_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
@@ -547,9 +547,9 @@ def enqueue_edit(
|
|||||||
try:
|
try:
|
||||||
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
|
job = service.enqueue_edit(db, user.id, source_job, spec, display_name)
|
||||||
except quota.QuotaExceeded as e:
|
except quota.QuotaExceeded as e:
|
||||||
raise HTTPException(status_code=422, detail=_quota_detail(e))
|
raise HTTPException(status_code=422, detail=_quota_detail(e)) from e
|
||||||
except service.EditError as e:
|
except service.EditError as e:
|
||||||
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason})
|
raise HTTPException(status_code=400, detail={"code": "edit", "reason": e.reason}) from e
|
||||||
return _serialize_job(db, job)
|
return _serialize_job(db, job)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -133,7 +133,7 @@ def get_feed(
|
|||||||
# Expose each key column on the row so we can build the next cursor from the last item.
|
# Expose each key column on the row so we can build the next cursor from the last item.
|
||||||
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
||||||
if vals is not None:
|
if vals is not None:
|
||||||
for k, v in zip(keys, vals):
|
for k, v in zip(keys, vals, strict=True):
|
||||||
k["value"] = v
|
k["value"] = v
|
||||||
if ranked and window is not None:
|
if ranked and window is not None:
|
||||||
rows = windowed_relevance_rows(db, query, keys, window, cursor_id, limit + 1)
|
rows = windowed_relevance_rows(db, query, keys, window, cursor_id, limit + 1)
|
||||||
@@ -431,7 +431,7 @@ def get_video_detail(
|
|||||||
):
|
):
|
||||||
items = yt.get_videos([video_id])
|
items = yt.get_videos([video_id])
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
if not items:
|
if not items:
|
||||||
raise not_found("video")
|
raise not_found("video")
|
||||||
|
|
||||||
|
|||||||
@@ -342,7 +342,7 @@ def sync_youtube(
|
|||||||
return {"ok": True, **pull_user_playlists(db, user)}
|
return {"ok": True, **pull_user_playlists(db, user)}
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
|
|
||||||
|
|
||||||
@router.post("/{playlist_id}/revert-youtube")
|
@router.post("/{playlist_id}/revert-youtube")
|
||||||
@@ -369,10 +369,10 @@ def revert_youtube(
|
|||||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PULL):
|
||||||
return {"ok": True, **repull_playlist(db, user, pl)}
|
return {"ok": True, **repull_playlist(db, user, pl)}
|
||||||
except PlaylistGone:
|
except PlaylistGone:
|
||||||
raise not_found("playlist")
|
raise not_found("playlist") from None
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc, playlist=True) from exc
|
||||||
|
|
||||||
|
|
||||||
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
def _pushable(db: Session, user: User, playlist_id: int) -> Playlist:
|
||||||
@@ -398,7 +398,7 @@ def push_plan(
|
|||||||
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH):
|
||||||
plan = plan_merge(db, user, pl)
|
plan = plan_merge(db, user, pl)
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc, playlist=True) from exc
|
||||||
plan["remaining_today"] = quota.remaining_today(db)
|
plan["remaining_today"] = quota.remaining_today(db)
|
||||||
plan["affordable"] = plan["units_estimate"] <= plan["remaining_today"]
|
plan["affordable"] = plan["units_estimate"] <= plan["remaining_today"]
|
||||||
return plan
|
return plan
|
||||||
@@ -436,10 +436,10 @@ def push(
|
|||||||
raise _refused(429, "quota_exceeded")
|
raise _refused(429, "quota_exceeded")
|
||||||
return {"ok": True, **merge_playlist(db, user, pl, live=live)}
|
return {"ok": True, **merge_playlist(db, user, pl, live=live)}
|
||||||
except PlaylistGone:
|
except PlaylistGone:
|
||||||
raise not_found("playlist")
|
raise not_found("playlist") from None
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc, playlist=True) from exc
|
||||||
|
|
||||||
|
|
||||||
@router.get("/{playlist_id}")
|
@router.get("/{playlist_id}")
|
||||||
@@ -539,7 +539,7 @@ def delete_playlist(
|
|||||||
with YouTubeClient(db, user) as yt:
|
with YouTubeClient(db, user) as yt:
|
||||||
yt.delete_playlist(pl.yt_playlist_id)
|
yt.delete_playlist(pl.yt_playlist_id)
|
||||||
except YouTubeError as exc:
|
except YouTubeError as exc:
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc, playlist=True) from exc
|
||||||
# The playlist first, as every item write: the cascade deletes the items before the list row.
|
# The playlist first, as every item write: the cascade deletes the items before the list row.
|
||||||
lock_playlists(db, pl.id)
|
lock_playlists(db, pl.id)
|
||||||
db.delete(pl) # items cascade
|
db.delete(pl) # items cascade
|
||||||
|
|||||||
@@ -428,7 +428,7 @@ def create_collection(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except (PlexError, PlexNotConfigured) as e:
|
except (PlexError, PlexNotConfigured) as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise _write_or_pending_error(wrote, e)
|
raise _write_or_pending_error(wrote, e) from e
|
||||||
return _collection_card(col)
|
return _collection_card(col)
|
||||||
|
|
||||||
|
|
||||||
@@ -450,7 +450,7 @@ def collection_add_item(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except (PlexError, PlexNotConfigured) as e:
|
except (PlexError, PlexNotConfigured) as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise _write_or_pending_error(wrote, e)
|
raise _write_or_pending_error(wrote, e) from e
|
||||||
return _collection_card(col)
|
return _collection_card(col)
|
||||||
|
|
||||||
|
|
||||||
@@ -472,7 +472,7 @@ def collection_remove_item(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except (PlexError, PlexNotConfigured) as e:
|
except (PlexError, PlexNotConfigured) as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise _write_or_pending_error(wrote, e)
|
raise _write_or_pending_error(wrote, e) from e
|
||||||
return _collection_card(col)
|
return _collection_card(col)
|
||||||
|
|
||||||
|
|
||||||
@@ -502,7 +502,7 @@ def collection_rename(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except (PlexError, PlexNotConfigured) as e:
|
except (PlexError, PlexNotConfigured) as e:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise _write_or_pending_error(wrote, e)
|
raise _write_or_pending_error(wrote, e) from e
|
||||||
return _collection_card(col)
|
return _collection_card(col)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -124,8 +124,14 @@ def _recently_added(
|
|||||||
# share one), so the merge doesn't depend on object identity staying alive.
|
# share one), so the merge doesn't depend on object identity staying alive.
|
||||||
top_movies = [o for _, kind, o in top if kind == "movie"]
|
top_movies = [o for _, kind, o in top if kind == "movie"]
|
||||||
top_shows = [o for _, kind, o in top if kind == "show"]
|
top_shows = [o for _, kind, o in top if kind == "show"]
|
||||||
mcards = {m.rating_key: c for m, c in zip(top_movies, _movie_cards(db, uid, top_movies, lang))}
|
mcards = {
|
||||||
scards = {s.rating_key: c for s, c in zip(top_shows, _show_cards(db, uid, top_shows, lang))}
|
m.rating_key: c
|
||||||
|
for m, c in zip(top_movies, _movie_cards(db, uid, top_movies, lang), strict=True)
|
||||||
|
}
|
||||||
|
scards = {
|
||||||
|
s.rating_key: c
|
||||||
|
for s, c in zip(top_shows, _show_cards(db, uid, top_shows, lang), strict=True)
|
||||||
|
}
|
||||||
return [mcards[o.rating_key] if kind == "movie" else scards[o.rating_key] for _, kind, o in top]
|
return [mcards[o.rating_key] if kind == "movie" else scards[o.rating_key] for _, kind, o in top]
|
||||||
|
|
||||||
|
|
||||||
@@ -351,10 +357,12 @@ def _because_you_watched(
|
|||||||
ordered_shows = [shows[rk] for rk in rks if rk in shows]
|
ordered_shows = [shows[rk] for rk in rks if rk in shows]
|
||||||
# rating_key is globally unique in Plex, so it keys the serialized cards back safely (no id()).
|
# rating_key is globally unique in Plex, so it keys the serialized cards back safely (no id()).
|
||||||
mcards = {
|
mcards = {
|
||||||
m.rating_key: c for m, c in zip(ordered_movies, _movie_cards(db, uid, ordered_movies, lang))
|
m.rating_key: c
|
||||||
|
for m, c in zip(ordered_movies, _movie_cards(db, uid, ordered_movies, lang), strict=True)
|
||||||
}
|
}
|
||||||
scards = {
|
scards = {
|
||||||
s.rating_key: c for s, c in zip(ordered_shows, _show_cards(db, uid, ordered_shows, lang))
|
s.rating_key: c
|
||||||
|
for s, c in zip(ordered_shows, _show_cards(db, uid, ordered_shows, lang), strict=True)
|
||||||
}
|
}
|
||||||
items: list[dict] = []
|
items: list[dict] = []
|
||||||
for rk in rks:
|
for rk in rks:
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel, field_validator
|
||||||
from sqlalchemy import func, select, update
|
from sqlalchemy import func, select, update
|
||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
@@ -53,11 +53,30 @@ def list_views(
|
|||||||
return [_serialize(v) for v in rows]
|
return [_serialize(v) for v in rows]
|
||||||
|
|
||||||
|
|
||||||
|
# A feed view's tag pick is read by more than the feed: the autotag job keeps every tag a view names
|
||||||
|
# (sync/autotag.py). Held here to the shape the feed sends — a list of tag ids, at most this many — so
|
||||||
|
# a hand-made body (`"tags": 5`, or 60k ids) can't break that job for every user (E32 review).
|
||||||
|
_MAX_VIEW_TAGS = 500
|
||||||
|
|
||||||
|
|
||||||
|
def _check_filters(filters: dict | None) -> dict | None:
|
||||||
|
tags = (filters or {}).get("tags")
|
||||||
|
if tags is None:
|
||||||
|
return filters
|
||||||
|
if not isinstance(tags, list) or len(tags) > _MAX_VIEW_TAGS:
|
||||||
|
raise ValueError(f"filters.tags must be a list of at most {_MAX_VIEW_TAGS} tag ids")
|
||||||
|
if any(type(t) is not int for t in tags): # `type`, not isinstance: True is not a tag id
|
||||||
|
raise ValueError("filters.tags must hold tag ids")
|
||||||
|
return filters
|
||||||
|
|
||||||
|
|
||||||
class CreateViewIn(BaseModel):
|
class CreateViewIn(BaseModel):
|
||||||
name: str
|
name: str
|
||||||
filters: dict
|
filters: dict
|
||||||
source: str = "feed"
|
source: str = "feed"
|
||||||
|
|
||||||
|
_filters = field_validator("filters")(_check_filters)
|
||||||
|
|
||||||
|
|
||||||
@router.post("")
|
@router.post("")
|
||||||
def create_view(
|
def create_view(
|
||||||
@@ -87,7 +106,7 @@ def create_view(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise refused(409, "view_name_taken", name=name)
|
raise refused(409, "view_name_taken", name=name) from None
|
||||||
return _serialize(view)
|
return _serialize(view)
|
||||||
|
|
||||||
|
|
||||||
@@ -98,6 +117,8 @@ class UpdateViewIn(BaseModel):
|
|||||||
filters: dict | None = None
|
filters: dict | None = None
|
||||||
is_default: bool | None = None
|
is_default: bool | None = None
|
||||||
|
|
||||||
|
_filters = field_validator("filters")(_check_filters)
|
||||||
|
|
||||||
|
|
||||||
@router.patch("/{view_id}")
|
@router.patch("/{view_id}")
|
||||||
def update_view(
|
def update_view(
|
||||||
@@ -135,7 +156,7 @@ def update_view(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise refused(409, "view_name_taken", name=view.name)
|
raise refused(409, "view_name_taken", name=view.name) from None
|
||||||
return _serialize(view)
|
return _serialize(view)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ def run_job_now(
|
|||||||
try:
|
try:
|
||||||
result = trigger_job(job_id, actor_id=user.id)
|
result = trigger_job(job_id, actor_id=user.id)
|
||||||
except KeyError:
|
except KeyError:
|
||||||
raise not_found("job")
|
raise not_found("job") from None
|
||||||
if result == "already_running":
|
if result == "already_running":
|
||||||
raise refused(409, "job_running")
|
raise refused(409, "job_running")
|
||||||
return {"ok": True, "id": job_id, "started": True}
|
return {"ok": True, "id": job_id, "started": True}
|
||||||
|
|||||||
@@ -331,7 +331,7 @@ def search_youtube(
|
|||||||
# bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1).
|
# bare 500 hidden behind the FE's generic "search failed" (same rationale as B-core-1).
|
||||||
try:
|
try:
|
||||||
collected.extend(_ingest_candidates(db, yt, user, candidates))
|
collected.extend(_ingest_candidates(db, yt, user, candidates))
|
||||||
except Exception as exc: # noqa: BLE001 — any ingest failure, not just YouTubeError
|
except Exception as exc: # any ingest failure, not just YouTubeError
|
||||||
db.rollback()
|
db.rollback()
|
||||||
# The term is NOT in this line. A search string is exactly what the privacy
|
# The term is NOT in this line. A search string is exactly what the privacy
|
||||||
# policy keeps out of the log unless the user turned Teljes/TRACE on for
|
# policy keeps out of the log unless the user turned Teljes/TRACE on for
|
||||||
|
|||||||
@@ -39,7 +39,7 @@ def sync_subscriptions(user: User = Depends(require_human), db: Session = Depend
|
|||||||
# The onboarding wizard's first call: an unguarded failure here was a raw 500 and the
|
# The onboarding wizard's first call: an unguarded failure here was a raw 500 and the
|
||||||
# blocking "Server error" dialog, with nothing the user could act on (E9 F1).
|
# blocking "Server error" dialog, with nothing the user could act on (E9 F1).
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise youtube_refusal(exc)
|
raise youtube_refusal(exc) from exc
|
||||||
result["quota_used_estimate"] = m.used
|
result["quota_used_estimate"] = m.used
|
||||||
result["quota_remaining_today"] = m.remaining
|
result["quota_remaining_today"] = m.remaining
|
||||||
return {"ok": True, **result}
|
return {"ok": True, **result}
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ def create_tag(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise refused(409, "tag_name_taken", name=name)
|
raise refused(409, "tag_name_taken", name=name) from None
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"id": tag.id,
|
"id": tag.id,
|
||||||
@@ -113,7 +113,7 @@ def update_tag(
|
|||||||
db.commit()
|
db.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
raise refused(409, "tag_name_taken", name=tag.name)
|
raise refused(409, "tag_name_taken", name=tag.name) from None
|
||||||
return {
|
return {
|
||||||
"ok": True,
|
"ok": True,
|
||||||
"id": tag.id,
|
"id": tag.id,
|
||||||
|
|||||||
+29
-12
@@ -12,7 +12,7 @@ from sqlalchemy import and_, exists, func, select
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs, progress, sysconfig
|
from app import logs, progress, sysconfig
|
||||||
from app.models import Channel, ChannelTag, Tag, Video
|
from app.models import Channel, ChannelTag, SavedView, Tag, Video
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
@@ -288,18 +288,35 @@ def run_autotag_all(db: Session, only_missing: bool = False) -> dict:
|
|||||||
return {"channels_tagged": tagged, "orphan_tags_removed": removed}
|
return {"channels_tagged": tagged, "orphan_tags_removed": removed}
|
||||||
|
|
||||||
|
|
||||||
|
def _view_tag_ids(filters: object) -> list[int]:
|
||||||
|
"""The tag ids a stored view's filters name; anything else in that slot names none."""
|
||||||
|
tags = filters.get("tags") if isinstance(filters, dict) else None
|
||||||
|
if not isinstance(tags, list):
|
||||||
|
return []
|
||||||
|
return [t for t in tags if type(t) is int] # `type`, not isinstance: True is not a tag id
|
||||||
|
|
||||||
|
|
||||||
def _cleanup_orphan_system_tags(db: Session) -> int:
|
def _cleanup_orphan_system_tags(db: Session) -> int:
|
||||||
"""Delete system tags that no longer link to any channel (e.g. after a mapping change)."""
|
"""Delete system tags that no longer link to any channel (e.g. after a mapping change) — but not
|
||||||
orphans = (
|
one a saved feed view filters on. Views hold tag IDS, and a deleted tag comes back under a new id
|
||||||
db.execute(
|
when a channel earns it again, so the view's filter would point at nothing; the feed drops an
|
||||||
select(Tag).where(
|
unknown id, and the view showed the whole feed under its old name (E32 S2).
|
||||||
Tag.user_id.is_(None),
|
|
||||||
~exists().where(ChannelTag.tag_id == Tag.id),
|
The views are read defensively, and the kept ids are filtered here rather than in SQL: a view
|
||||||
)
|
stored before its route checked the shape (or written around it) must not raise for everyone's
|
||||||
)
|
job, nor push a `NOT IN` past the driver's bind-parameter limit (E32 review)."""
|
||||||
.scalars()
|
in_views = {
|
||||||
.all()
|
t
|
||||||
)
|
for (filters,) in db.execute(select(SavedView.filters).where(SavedView.source == "feed"))
|
||||||
|
for t in _view_tag_ids(filters)
|
||||||
|
}
|
||||||
|
orphans = [
|
||||||
|
tag
|
||||||
|
for tag in db.execute(
|
||||||
|
select(Tag).where(Tag.user_id.is_(None), ~exists().where(ChannelTag.tag_id == Tag.id))
|
||||||
|
).scalars()
|
||||||
|
if tag.id not in in_views
|
||||||
|
]
|
||||||
for tag in orphans:
|
for tag in orphans:
|
||||||
db.delete(tag)
|
db.delete(tag)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ from app.models import (
|
|||||||
)
|
)
|
||||||
from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
|
from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
|
||||||
from app.utils import now_utc as _now
|
from app.utils import now_utc as _now
|
||||||
from app.youtube.client import YouTubeClient
|
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
@@ -81,7 +81,15 @@ def explore_ingest_page(
|
|||||||
the organic deep backfill) is never touched."""
|
the organic deep backfill) is never touched."""
|
||||||
if not channel.uploads_playlist_id:
|
if not channel.uploads_playlist_id:
|
||||||
return {"ingested": 0, "next_page_token": None}
|
return {"ingested": 0, "next_page_token": None}
|
||||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
try:
|
||||||
|
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||||
|
except YouTubeError as exc:
|
||||||
|
# A channel with no uploads has an uploads id that YouTube answers as not found: an empty
|
||||||
|
# back-catalogue, not a failure (and not a "playlist gone" — the user picked a channel).
|
||||||
|
if not is_playlist_gone(exc):
|
||||||
|
raise
|
||||||
|
log.info("explore ingest channel=%s has no uploads playlist", channel.id)
|
||||||
|
return {"ingested": 0, "next_page_token": None}
|
||||||
stubs: list[dict] = []
|
stubs: list[dict] = []
|
||||||
for item in data.get("items", []):
|
for item in data.get("items", []):
|
||||||
stub = stub_from_playlist_item(item, channel.id)
|
stub = stub_from_playlist_item(item, channel.id)
|
||||||
|
|||||||
@@ -20,9 +20,9 @@ Quota: phase 1 costs ~(flagged / 50) units, phase 2 ~(batch / 50). Both respect
|
|||||||
backfill reserve as the other jobs so interactive syncs never starve.
|
backfill reserve as the other jobs so interactive syncs never starve.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import timedelta
|
from datetime import datetime, timedelta
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import or_, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs, progress, quota, sysconfig
|
from app import logs, progress, quota, sysconfig
|
||||||
@@ -129,14 +129,14 @@ def _notify_removed(db: Session, impact: dict[int, list[dict]]) -> int:
|
|||||||
return len(impact)
|
return len(impact)
|
||||||
|
|
||||||
|
|
||||||
def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
|
def _recheck_flagged(db: Session, yt: YouTubeClient, now: datetime) -> dict:
|
||||||
"""Phase 1: recover or hard-delete videos already flagged unavailable."""
|
"""Phase 1: recover or hard-delete videos already flagged unavailable. `now` is the run's:
|
||||||
|
phase 2 skips a video stamped with it, so one recovered here is not fetched again."""
|
||||||
flagged = db.execute(select(Video).where(Video.unavailable_since.is_not(None))).scalars().all()
|
flagged = db.execute(select(Video).where(Video.unavailable_since.is_not(None))).scalars().all()
|
||||||
if not flagged:
|
if not flagged:
|
||||||
return {"checked": 0, "recovered": 0, "deleted": 0, "notified": 0}
|
return {"checked": 0, "recovered": 0, "deleted": 0, "notified": 0}
|
||||||
progress.report(0, len(flagged), "recheck")
|
progress.report(0, len(flagged), "recheck")
|
||||||
items = {it["id"]: it for it in yt.get_videos([v.id for v in flagged])}
|
items = {it["id"]: it for it in yt.get_videos([v.id for v in flagged])}
|
||||||
now = _now()
|
|
||||||
recovered = 0
|
recovered = 0
|
||||||
to_delete: list[Video] = []
|
to_delete: list[Video] = []
|
||||||
for video in flagged:
|
for video in flagged:
|
||||||
@@ -173,7 +173,7 @@ def _recheck_flagged(db: Session, yt: YouTubeClient) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
def _revalidate_rolling(db: Session, yt: YouTubeClient, now: datetime) -> dict:
|
||||||
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
|
"""Phase 2: re-check the least-recently-checked currently-available videos and flag any
|
||||||
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
|
that have become unplayable (hidden immediately; deleted later if they don't recover)."""
|
||||||
batch = sysconfig.get_int(
|
batch = sysconfig.get_int(
|
||||||
@@ -181,7 +181,6 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
|||||||
) # admin override, else config default
|
) # admin override, else config default
|
||||||
flagged_new = 0
|
flagged_new = 0
|
||||||
checked = 0
|
checked = 0
|
||||||
now = _now()
|
|
||||||
# Process in videos.list-sized pages so we commit incrementally and stop on low quota.
|
# Process in videos.list-sized pages so we commit incrementally and stop on low quota.
|
||||||
while checked < batch:
|
while checked < batch:
|
||||||
if quota.backfill_reserve_reached():
|
if quota.backfill_reserve_reached():
|
||||||
@@ -190,6 +189,10 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
|||||||
db.execute(
|
db.execute(
|
||||||
select(Video)
|
select(Video)
|
||||||
.where(Video.unavailable_since.is_(None))
|
.where(Video.unavailable_since.is_(None))
|
||||||
|
# Not one this run already checked: with fewer available videos than the batch, the
|
||||||
|
# loop used to wrap around and re-fetch the same ones until `checked` reached the
|
||||||
|
# batch, spending quota on answers it already had (E32 S2).
|
||||||
|
.where(or_(Video.last_checked_at.is_(None), Video.last_checked_at < now))
|
||||||
# NULLS FIRST = never-checked first, then least-recently-checked; equivalent to the old
|
# NULLS FIRST = never-checked first, then least-recently-checked; equivalent to the old
|
||||||
# `(last_checked_at IS NULL) DESC, last_checked_at ASC` but in the exact shape of the
|
# `(last_checked_at IS NULL) DESC, last_checked_at ASC` but in the exact shape of the
|
||||||
# partial ix_videos_recheck_order (migration 0069, PERF-18) so it rides that index.
|
# partial ix_videos_recheck_order (migration 0069, PERF-18) so it rides that index.
|
||||||
@@ -220,10 +223,13 @@ def run_maintenance(db: Session) -> dict:
|
|||||||
user = get_service_user(db)
|
user = get_service_user(db)
|
||||||
if user is None:
|
if user is None:
|
||||||
return {"reason": "no service user"}
|
return {"reason": "no service user"}
|
||||||
|
# One `now` for the whole run: each phase taking its own let a video phase 1 had just recovered
|
||||||
|
# (stamped with phase 1's) pass phase 2's "not checked this run" test and be fetched twice.
|
||||||
|
now = _now()
|
||||||
with YouTubeClient(db, user) as yt:
|
with YouTubeClient(db, user) as yt:
|
||||||
phase1 = _recheck_flagged(db, yt)
|
phase1 = _recheck_flagged(db, yt, now)
|
||||||
phase2 = (
|
phase2 = (
|
||||||
_revalidate_rolling(db, yt)
|
_revalidate_rolling(db, yt, now)
|
||||||
if not quota.backfill_reserve_reached()
|
if not quota.backfill_reserve_reached()
|
||||||
else {"checked": 0, "flagged": 0, "skipped": "low quota"}
|
else {"checked": 0, "flagged": 0, "skipped": "low quota"}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,89 @@
|
|||||||
|
"""The 3-way merge of a YouTube-linked playlist, as a pure function: what each side gains and loses
|
||||||
|
so neither loses an item, the merged order, and the reorder moves YouTube needs to reach it. No I/O —
|
||||||
|
`sync/playlists.py` reads both sides, calls `merge_plan`, and writes the outcome."""
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
|
||||||
|
|
||||||
|
def _reorder_moves(current: list[str], desired: list[str]) -> int:
|
||||||
|
"""Count the playlistItems.update moves an insertion-sort would need to turn
|
||||||
|
`current` order into `desired` order (both already the same set of video ids).
|
||||||
|
Items already in place are skipped; this matches what apply_push actually executes."""
|
||||||
|
model = list(current)
|
||||||
|
moves = 0
|
||||||
|
for i, want in enumerate(desired):
|
||||||
|
if i < len(model) and model[i] == want:
|
||||||
|
continue
|
||||||
|
j = model.index(want, i)
|
||||||
|
model.pop(j)
|
||||||
|
model.insert(i, want)
|
||||||
|
moves += 1
|
||||||
|
return moves
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MergePlan:
|
||||||
|
"""The 3-way merge of a linked playlist: what each side gains and loses so neither side
|
||||||
|
loses an item, plus the merged order. `final` is the id set both sides end up with."""
|
||||||
|
|
||||||
|
final: list[str] # merged order (local order, YouTube-only adds appended)
|
||||||
|
add_to_yt: list[str] # local-adds to push out to YouTube
|
||||||
|
add_to_local: list[str] # YouTube-adds to pull into Siftlode
|
||||||
|
remove_from_yt: list[str] # items deleted locally that are still on YouTube
|
||||||
|
remove_from_local: list[str] # items deleted on YouTube that are still local
|
||||||
|
reorders: int # estimated YouTube playlistItems.update moves to reach `final`
|
||||||
|
baseline_known: bool
|
||||||
|
|
||||||
|
|
||||||
|
def merge_plan(
|
||||||
|
baseline: list[str] | None,
|
||||||
|
local: list[str],
|
||||||
|
youtube: list[str],
|
||||||
|
unavailable: set[str] | None = None,
|
||||||
|
) -> MergePlan:
|
||||||
|
"""Pure 3-way merge. Classify each id against the last-synced `baseline`:
|
||||||
|
- in local but not youtube, not deleted → add to YouTube (a local-add)
|
||||||
|
- in youtube but not local, not deleted → add to Siftlode (a YouTube-add)
|
||||||
|
- in baseline but gone from local → a genuine local delete → propagate to YouTube
|
||||||
|
- in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode
|
||||||
|
With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both
|
||||||
|
sides (never delete) — the lossless default. Merged order: local order first, YouTube-only
|
||||||
|
additions appended in their YouTube order.
|
||||||
|
`unavailable` = known un-ingestable-but-live YouTube ids: dropped from the YouTube side up
|
||||||
|
front so they're never re-classified as a YouTube-add (they can't ingest) nor as a local
|
||||||
|
delete (they were never local) — otherwise such a video would make the playlist never
|
||||||
|
converge. They stay in place on YouTube, untouched by this merge."""
|
||||||
|
un = set(unavailable or ())
|
||||||
|
L = list(dict.fromkeys(local))
|
||||||
|
Y = [v for v in dict.fromkeys(youtube) if v not in un]
|
||||||
|
Lset, Yset = set(L), set(Y)
|
||||||
|
if baseline is None:
|
||||||
|
add_to_yt = [v for v in L if v not in Yset]
|
||||||
|
add_to_local = [v for v in Y if v not in Lset]
|
||||||
|
remove_from_yt: list[str] = []
|
||||||
|
remove_from_local: list[str] = []
|
||||||
|
final = L + [v for v in Y if v not in Lset]
|
||||||
|
else:
|
||||||
|
B = set(baseline)
|
||||||
|
deleted_local = B - Lset # was synced, removed on our side
|
||||||
|
deleted_yt = B - Yset # was synced, removed on YouTube's side
|
||||||
|
gone = deleted_local | deleted_yt
|
||||||
|
remove_from_yt = [v for v in Y if v in deleted_local]
|
||||||
|
remove_from_local = [v for v in L if v in deleted_yt]
|
||||||
|
add_to_yt = [v for v in L if v not in Yset and v not in gone]
|
||||||
|
add_to_local = [v for v in Y if v not in Lset and v not in gone]
|
||||||
|
final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone]
|
||||||
|
final_set = set(final)
|
||||||
|
# On YouTube, after membership reconciliation the order is current-kept + inserts appended;
|
||||||
|
# the reorder pass then moves items into `final`.
|
||||||
|
after_membership = [v for v in Y if v in final_set] + add_to_yt
|
||||||
|
reorders = _reorder_moves(after_membership, final)
|
||||||
|
return MergePlan(
|
||||||
|
final=final,
|
||||||
|
add_to_yt=add_to_yt,
|
||||||
|
add_to_local=add_to_local,
|
||||||
|
remove_from_yt=remove_from_yt,
|
||||||
|
remove_from_local=remove_from_local,
|
||||||
|
reorders=reorders,
|
||||||
|
baseline_known=baseline is not None,
|
||||||
|
)
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists
|
"""YouTube playlist sync, both directions. Read: mirror each user's own YouTube playlists
|
||||||
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync
|
into local `source='youtube'` playlists (kept fresh by the scheduler). Write-back: two-way sync
|
||||||
local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope.
|
local edits with YouTube (`plan_merge` / `merge_playlist`), behind the optional write scope; the
|
||||||
|
pure 3-way merge they run is `sync/playlist_merge.py`.
|
||||||
|
|
||||||
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
|
YouTube's special Watch Later / History playlists are not exposed by the Data API and are
|
||||||
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
|
never synced. Videos in a playlist that aren't in our shared catalog yet are fetched and
|
||||||
@@ -10,7 +11,7 @@ import hashlib
|
|||||||
import json
|
import json
|
||||||
from collections.abc import Callable, Iterator
|
from collections.abc import Callable, Iterator
|
||||||
from contextlib import AbstractContextManager, contextmanager
|
from contextlib import AbstractContextManager, contextmanager
|
||||||
from dataclasses import dataclass
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from typing import TypeVar
|
from typing import TypeVar
|
||||||
|
|
||||||
@@ -21,9 +22,11 @@ from sqlalchemy.pool import NullPool
|
|||||||
from app import logs, progress, quota
|
from app import logs, progress, quota
|
||||||
from app.auth import has_read_scope
|
from app.auth import has_read_scope
|
||||||
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
from app.models import Channel, Playlist, PlaylistItem, User, Video
|
||||||
|
from app.sync.playlist_merge import merge_plan
|
||||||
from app.sync.runner import token_users
|
from app.sync.runner import token_users
|
||||||
|
from app.sync.subscriptions import blank_listing, refuse_blank_listing, vanished
|
||||||
from app.sync.videos import apply_video_details, parse_dt
|
from app.sync.videos import apply_video_details, parse_dt
|
||||||
from app.youtube.client import YouTubeClient, YouTubeError
|
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
# Cost of a single YouTube write op (playlists/playlistItems insert/update/delete).
|
||||||
@@ -316,6 +319,24 @@ def user_sync_claim(db: Session, user_id: int) -> AbstractContextManager[bool]:
|
|||||||
return _advisory_claim(db, _USER_SYNC_LOCK_NS, user_id)
|
return _advisory_claim(db, _USER_SYNC_LOCK_NS, user_id)
|
||||||
|
|
||||||
|
|
||||||
|
def _mirror_listing(yt: YouTubeClient, ytid: str, count: object, pl_id: int) -> list[str] | None:
|
||||||
|
"""A mirror's video ids from YouTube, or None when there is nothing to believe: the list was
|
||||||
|
deleted between the pull's listing and this read (the next pull drops the mirror as vanished —
|
||||||
|
not a reason to fail every other list, E32 review round 2), or YouTube listed no items for a
|
||||||
|
list it counts as non-empty (`blank_listing`)."""
|
||||||
|
try:
|
||||||
|
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
||||||
|
except YouTubeError as exc:
|
||||||
|
if not is_playlist_gone(exc):
|
||||||
|
raise
|
||||||
|
log.info("YouTube playlist of %s vanished mid-pull; skipped", pl_id)
|
||||||
|
return None
|
||||||
|
if blank_listing(ids, count):
|
||||||
|
log.warning("YouTube listed no items for non-empty playlist %s; kept", pl_id)
|
||||||
|
return None
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
def pull_user_playlists(db: Session, user: User) -> dict:
|
def pull_user_playlists(db: Session, user: User) -> dict:
|
||||||
"""The mirror pull itself; the caller holds `user_sync_claim`."""
|
"""The mirror pull itself; the caller holds `user_sync_claim`."""
|
||||||
synced = 0
|
synced = 0
|
||||||
@@ -337,12 +358,10 @@ def pull_user_playlists(db: Session, user: User) -> dict:
|
|||||||
Playlist.source == "local",
|
Playlist.source == "local",
|
||||||
Playlist.yt_playlist_id.is_not(None),
|
Playlist.yt_playlist_id.is_not(None),
|
||||||
)
|
)
|
||||||
)
|
).scalars()
|
||||||
.scalars()
|
|
||||||
.all()
|
|
||||||
)
|
)
|
||||||
# Drop mirrors whose YouTube playlist no longer exists — locked first, as every item write.
|
# Drop vanished mirrors (none on an empty answer) — locked first, as every item write.
|
||||||
gone = [pl for ytid, pl in existing.items() if ytid not in yt_ids]
|
gone = vanished(existing, yt_ids)
|
||||||
if gone:
|
if gone:
|
||||||
lock_playlists(db, *(pl.id for pl in gone))
|
lock_playlists(db, *(pl.id for pl in gone))
|
||||||
for pl in gone:
|
for pl in gone:
|
||||||
@@ -376,7 +395,10 @@ def pull_user_playlists(db: Session, user: User) -> dict:
|
|||||||
# The user is pushing or re-pulling this very list: it writes the same rows.
|
# The user is pushing or re-pulling this very list: it writes the same rows.
|
||||||
# The next scheduled run refreshes it.
|
# The next scheduled run refreshes it.
|
||||||
continue
|
continue
|
||||||
ids = list(yt.iter_my_playlist_video_ids(ytid))
|
ids = _mirror_listing(yt, ytid, p.get("item_count"), pl.id)
|
||||||
|
if ids is None:
|
||||||
|
db.rollback() # a mirror this run just created is not kept empty either
|
||||||
|
continue
|
||||||
# Ingest first — it commits — so the row lock below is held from the dirty check
|
# Ingest first — it commits — so the row lock below is held from the dirty check
|
||||||
# to the commit that replaces the items.
|
# to the commit that replaces the items.
|
||||||
_ensure_videos(db, yt, ids)
|
_ensure_videos(db, yt, ids)
|
||||||
@@ -408,90 +430,6 @@ def _local_video_ids(db: Session, pl: Playlist) -> list[str]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _reorder_moves(current: list[str], desired: list[str]) -> int:
|
|
||||||
"""Count the playlistItems.update moves an insertion-sort would need to turn
|
|
||||||
`current` order into `desired` order (both already the same set of video ids).
|
|
||||||
Items already in place are skipped; this matches what apply_push actually executes."""
|
|
||||||
model = list(current)
|
|
||||||
moves = 0
|
|
||||||
for i, want in enumerate(desired):
|
|
||||||
if i < len(model) and model[i] == want:
|
|
||||||
continue
|
|
||||||
j = model.index(want, i)
|
|
||||||
model.pop(j)
|
|
||||||
model.insert(i, want)
|
|
||||||
moves += 1
|
|
||||||
return moves
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class MergePlan:
|
|
||||||
"""The 3-way merge of a linked playlist: what each side gains and loses so neither side
|
|
||||||
loses an item, plus the merged order. `final` is the id set both sides end up with."""
|
|
||||||
|
|
||||||
final: list[str] # merged order (local order, YouTube-only adds appended)
|
|
||||||
add_to_yt: list[str] # local-adds to push out to YouTube
|
|
||||||
add_to_local: list[str] # YouTube-adds to pull into Siftlode
|
|
||||||
remove_from_yt: list[str] # items deleted locally that are still on YouTube
|
|
||||||
remove_from_local: list[str] # items deleted on YouTube that are still local
|
|
||||||
reorders: int # estimated YouTube playlistItems.update moves to reach `final`
|
|
||||||
baseline_known: bool
|
|
||||||
|
|
||||||
|
|
||||||
def merge_plan(
|
|
||||||
baseline: list[str] | None,
|
|
||||||
local: list[str],
|
|
||||||
youtube: list[str],
|
|
||||||
unavailable: set[str] | None = None,
|
|
||||||
) -> MergePlan:
|
|
||||||
"""Pure 3-way merge. Classify each id against the last-synced `baseline`:
|
|
||||||
- in local but not youtube, not deleted → add to YouTube (a local-add)
|
|
||||||
- in youtube but not local, not deleted → add to Siftlode (a YouTube-add)
|
|
||||||
- in baseline but gone from local → a genuine local delete → propagate to YouTube
|
|
||||||
- in baseline but gone from youtube → a genuine YouTube delete → propagate to Siftlode
|
|
||||||
With NO baseline (first-ever merge) we can't tell an add from a delete, so we UNION both
|
|
||||||
sides (never delete) — the lossless default. Merged order: local order first, YouTube-only
|
|
||||||
additions appended in their YouTube order.
|
|
||||||
`unavailable` = known un-ingestable-but-live YouTube ids: dropped from the YouTube side up
|
|
||||||
front so they're never re-classified as a YouTube-add (they can't ingest) nor as a local
|
|
||||||
delete (they were never local) — otherwise such a video would make the playlist never
|
|
||||||
converge. They stay in place on YouTube, untouched by this merge."""
|
|
||||||
un = set(unavailable or ())
|
|
||||||
L = list(dict.fromkeys(local))
|
|
||||||
Y = [v for v in dict.fromkeys(youtube) if v not in un]
|
|
||||||
Lset, Yset = set(L), set(Y)
|
|
||||||
if baseline is None:
|
|
||||||
add_to_yt = [v for v in L if v not in Yset]
|
|
||||||
add_to_local = [v for v in Y if v not in Lset]
|
|
||||||
remove_from_yt: list[str] = []
|
|
||||||
remove_from_local: list[str] = []
|
|
||||||
final = L + [v for v in Y if v not in Lset]
|
|
||||||
else:
|
|
||||||
B = set(baseline)
|
|
||||||
deleted_local = B - Lset # was synced, removed on our side
|
|
||||||
deleted_yt = B - Yset # was synced, removed on YouTube's side
|
|
||||||
gone = deleted_local | deleted_yt
|
|
||||||
remove_from_yt = [v for v in Y if v in deleted_local]
|
|
||||||
remove_from_local = [v for v in L if v in deleted_yt]
|
|
||||||
add_to_yt = [v for v in L if v not in Yset and v not in gone]
|
|
||||||
add_to_local = [v for v in Y if v not in Lset and v not in gone]
|
|
||||||
final = [v for v in L if v not in gone] + [v for v in Y if v not in Lset and v not in gone]
|
|
||||||
final_set = set(final)
|
|
||||||
# On YouTube, after membership reconciliation the order is current-kept + inserts appended;
|
|
||||||
# the reorder pass then moves items into `final`.
|
|
||||||
after_membership = [v for v in Y if v in final_set] + add_to_yt
|
|
||||||
reorders = _reorder_moves(after_membership, final)
|
|
||||||
return MergePlan(
|
|
||||||
final=final,
|
|
||||||
add_to_yt=add_to_yt,
|
|
||||||
add_to_local=add_to_local,
|
|
||||||
remove_from_yt=remove_from_yt,
|
|
||||||
remove_from_local=remove_from_local,
|
|
||||||
reorders=reorders,
|
|
||||||
baseline_known=baseline is not None,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
||||||
"""Compute (without writing) what a two-way sync of this playlist would do: create vs
|
"""Compute (without writing) what a two-way sync of this playlist would do: create vs
|
||||||
update, how many items each side gains/loses, reorders, and the quota estimate (YouTube
|
update, how many items each side gains/loses, reorders, and the quota estimate (YouTube
|
||||||
@@ -519,6 +457,7 @@ def plan_merge(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|||||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||||
rename = bool(snippet) and (snippet.get("title") or "") != pl.name
|
rename = bool(snippet) and (snippet.get("title") or "") != pl.name
|
||||||
yt_ids = [it["video_id"] for it in current]
|
yt_ids = [it["video_id"] for it in current]
|
||||||
|
refuse_blank_listing(yt_ids, (snippet or {}).get("item_count"))
|
||||||
mp = merge_plan(parse_baseline(pl), local, yt_ids, parse_unavailable(pl))
|
mp = merge_plan(parse_baseline(pl), local, yt_ids, parse_unavailable(pl))
|
||||||
ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0)
|
ops = len(mp.add_to_yt) + len(mp.remove_from_yt) + mp.reorders + (1 if rename else 0)
|
||||||
return {
|
return {
|
||||||
@@ -665,6 +604,8 @@ def merge_playlist(db: Session, user: User, pl: Playlist, *, live=None) -> dict:
|
|||||||
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
current = list(yt.iter_playlist_items_with_ids(pl.yt_playlist_id))
|
||||||
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
item_id_by_vid = {it["video_id"]: it["item_id"] for it in current}
|
||||||
yt_ids = [it["video_id"] for it in current]
|
yt_ids = [it["video_id"] for it in current]
|
||||||
|
# A merge would read "YouTube lists none of them" as "YouTube deleted them all".
|
||||||
|
refuse_blank_listing(yt_ids, (snippet or {}).get("item_count"))
|
||||||
# Re-attempt ingestion of ids previously marked un-ingestable that are still on YouTube: a
|
# Re-attempt ingestion of ids previously marked un-ingestable that are still on YouTube: a
|
||||||
# video that has since become available (e.g. private→public) should re-enter the merge, not
|
# video that has since become available (e.g. private→public) should re-enter the merge, not
|
||||||
# stay skipped forever. Those that still won't ingest remain filtered (`effective_unavailable`).
|
# stay skipped forever. Those that still won't ingest remain filtered (`effective_unavailable`).
|
||||||
@@ -803,8 +744,11 @@ def repull_playlist(db: Session, user: User, pl: Playlist) -> dict:
|
|||||||
with YouTubeClient(db, user) as yt:
|
with YouTubeClient(db, user) as yt:
|
||||||
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
snippet = yt.get_playlist_snippet(pl.yt_playlist_id)
|
||||||
if snippet is None:
|
if snippet is None:
|
||||||
raise YouTubeError("Playlist no longer exists on YouTube")
|
raise YouTubeError(
|
||||||
|
"Playlist no longer exists on YouTube", reason="playlistNotFound", path="playlists"
|
||||||
|
)
|
||||||
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
ids = list(yt.iter_my_playlist_video_ids(pl.yt_playlist_id))
|
||||||
|
refuse_blank_listing(ids, snippet.get("item_count"))
|
||||||
_ensure_videos(db, yt, ids)
|
_ensure_videos(db, yt, ids)
|
||||||
_lock_or_gone(db, pl)
|
_lock_or_gone(db, pl)
|
||||||
pl.name = snippet.get("title") or pl.name
|
pl.name = snippet.get("title") or pl.name
|
||||||
|
|||||||
@@ -15,8 +15,6 @@ from sqlalchemy import func, or_, select, update
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import logs, progress, quota, sysconfig
|
from app import logs, progress, quota, sysconfig
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
|
||||||
from app.db import SessionLocal
|
from app.db import SessionLocal
|
||||||
from app.models import Channel, OAuthToken, Subscription, User
|
from app.models import Channel, OAuthToken, Subscription, User
|
||||||
from app.sync.explore import followed_channel
|
from app.sync.explore import followed_channel
|
||||||
@@ -35,6 +33,8 @@ from app.sync.videos import (
|
|||||||
from app.youtube.client import YouTubeClient, YouTubeError, is_quota_refusal
|
from app.youtube.client import YouTubeClient, YouTubeError, is_quota_refusal
|
||||||
from app.youtube.rss import RssError, fetch_channel_feed
|
from app.youtube.rss import RssError, fetch_channel_feed
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
|
# RSS feeds are free and network-bound, so fetch many at once; DB writes stay single-threaded.
|
||||||
RSS_POLL_WORKERS = 16
|
RSS_POLL_WORKERS = 16
|
||||||
# Failed channels named in a poll's summary warning; the rest are counted by reason.
|
# Failed channels named in a poll's summary warning; the rest are counted by reason.
|
||||||
@@ -267,7 +267,7 @@ def run_feed_fallback(db: Session, channel_ids: list[str]) -> dict[str, int]:
|
|||||||
channel.feed_fallback_at = datetime.now(timezone.utc)
|
channel.feed_fallback_at = datetime.now(timezone.utc)
|
||||||
db.add(channel)
|
db.add(channel)
|
||||||
db.commit()
|
db.commit()
|
||||||
except Exception: # noqa: BLE001 — one channel must not stop the rest
|
except Exception: # one channel must not stop the rest
|
||||||
db.rollback()
|
db.rollback()
|
||||||
log.exception(
|
log.exception(
|
||||||
"Feed fallback could not store channel %s",
|
"Feed fallback could not store channel %s",
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
"""Import the authenticated user's YouTube subscriptions and channel metadata."""
|
||||||
|
|
||||||
from collections.abc import Iterable
|
from collections.abc import Collection, Iterable
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
from sqlalchemy import or_, select
|
from sqlalchemy import or_, select
|
||||||
@@ -9,7 +9,7 @@ from sqlalchemy.orm import Session
|
|||||||
from app import logs
|
from app import logs
|
||||||
from app.models import Channel, Subscription, User
|
from app.models import Channel, Subscription, User
|
||||||
from app.utils import normalize_description
|
from app.utils import normalize_description
|
||||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
@@ -115,10 +115,37 @@ def should_prune(fetched_count: int, existing_count: int) -> bool:
|
|||||||
has — taking each row's `deep_requested` flag with it — in a single run. Someone who genuinely
|
has — taking each row's `deep_requested` flag with it — in a single run. Someone who genuinely
|
||||||
unsubscribed from everything simply keeps their rows until a sync that returns something.
|
unsubscribed from everything simply keeps their rows until a sync that returns something.
|
||||||
|
|
||||||
Pure so it can be tested without a database; `import_subscriptions` is the only caller."""
|
Pure so it can be tested without a database; its callers are `import_subscriptions` and, through
|
||||||
|
`vanished`, every mirror that drops what the remote no longer lists."""
|
||||||
return fetched_count > 0 or existing_count == 0
|
return fetched_count > 0 or existing_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
def vanished[K, V](existing: dict[K, V], listed: Collection[K]) -> list[V]:
|
||||||
|
"""The local mirrors whose key the remote listing no longer has — none at all when the listing
|
||||||
|
is an empty answer that contradicts them (`should_prune`). The YouTube playlist mirrors use it:
|
||||||
|
without it, one blip dropped every mirror (E32 S2)."""
|
||||||
|
if not should_prune(len(listed), len(existing)):
|
||||||
|
return []
|
||||||
|
return [row for key, row in existing.items() if key not in listed]
|
||||||
|
|
||||||
|
|
||||||
|
def blank_listing(listed: Collection, count: object) -> bool:
|
||||||
|
"""The same rule one level down: no members from a container whose own listing counts some (a
|
||||||
|
playlist's `itemCount`) is a blip, not a list emptied. Replacing or merging on it deleted every
|
||||||
|
mirrored item and reset the merge baseline (E32 review)."""
|
||||||
|
return not listed and isinstance(count, int) and count > 0
|
||||||
|
|
||||||
|
|
||||||
|
def refuse_blank_listing(listed: Collection, count: object) -> None:
|
||||||
|
"""`blank_listing` for a user-started pull or push: refused as retryable ("try again")."""
|
||||||
|
if blank_listing(listed, count):
|
||||||
|
raise YouTubeError(
|
||||||
|
"YouTube listed no items for a playlist it counts as non-empty",
|
||||||
|
reason="emptyAnswer",
|
||||||
|
path="playlistItems",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def import_subscriptions(db: Session, user: User) -> dict:
|
def import_subscriptions(db: Session, user: User) -> dict:
|
||||||
"""Pull all subscriptions for `user`, upsert channels + subscription links, prune
|
"""Pull all subscriptions for `user`, upsert channels + subscription links, prune
|
||||||
channels the user has unsubscribed from on YouTube, and fetch channel details for
|
channels the user has unsubscribed from on YouTube, and fetch channel details for
|
||||||
|
|||||||
@@ -11,12 +11,14 @@ from sqlalchemy import and_, func, or_, select, update
|
|||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app import progress, quota, sysconfig
|
from app import logs, progress, quota, sysconfig
|
||||||
from app.models import LIVE_OR_UPCOMING, Channel, Video
|
from app.models import LIVE_OR_UPCOMING, Channel, Video
|
||||||
from app.titles import normalize_title
|
from app.titles import normalize_title
|
||||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail, is_playlist_gone
|
||||||
from app.youtube.shorts import make_client, probe_is_short
|
from app.youtube.shorts import make_client, probe_is_short
|
||||||
|
|
||||||
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
_DURATION_RE = re.compile(r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
|
_DURATION_RE = re.compile(r"P(?:(\d+)D)?T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", re.IGNORECASE)
|
||||||
|
|
||||||
|
|
||||||
@@ -104,6 +106,20 @@ def stub_from_playlist_item(item: dict, channel_id: str) -> dict | None:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _uploads_page(yt: YouTubeClient, channel_id: str, uploads: str, page_token: str | None) -> dict:
|
||||||
|
"""One page of the channel's uploads. YouTube answers the uploads list of a channel with no
|
||||||
|
uploads at all as not found: an empty back-catalogue, so an empty last page — not a failure
|
||||||
|
that left `recent_synced_at` unstamped and charged the channel again on every run (E32 review).
|
||||||
|
Any other failure raises."""
|
||||||
|
try:
|
||||||
|
return yt.get_playlist_items(uploads, page_token)
|
||||||
|
except YouTubeError as exc:
|
||||||
|
if not is_playlist_gone(exc):
|
||||||
|
raise
|
||||||
|
log.info("backfill channel=%s has no uploads playlist", channel_id)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> int:
|
def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) -> int:
|
||||||
"""First pass: newest videos up to the configured count or age cutoff. Records a
|
"""First pass: newest videos up to the configured count or age cutoff. Records a
|
||||||
resume cursor for the later deep backfill."""
|
resume cursor for the later deep backfill."""
|
||||||
@@ -121,7 +137,7 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
|
|||||||
|
|
||||||
while True:
|
while True:
|
||||||
page_start = page_token # token that fetched THIS page (None = first page)
|
page_start = page_token # token that fetched THIS page (None = first page)
|
||||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token)
|
||||||
stopped_mid_page = False
|
stopped_mid_page = False
|
||||||
for item in data.get("items", []):
|
for item in data.get("items", []):
|
||||||
stub = stub_from_playlist_item(item, channel.id)
|
stub = stub_from_playlist_item(item, channel.id)
|
||||||
@@ -175,7 +191,7 @@ def backfill_channel_deep(
|
|||||||
for _ in range(max_pages):
|
for _ in range(max_pages):
|
||||||
if quota.backfill_reserve_reached():
|
if quota.backfill_reserve_reached():
|
||||||
break
|
break
|
||||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
data = _uploads_page(yt, channel.id, channel.uploads_playlist_id, page_token)
|
||||||
rows = [
|
rows = [
|
||||||
s
|
s
|
||||||
for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", []))
|
for s in (stub_from_playlist_item(it, channel.id) for it in data.get("items", []))
|
||||||
|
|||||||
@@ -60,7 +60,6 @@ _SHORT_LOWER = {
|
|||||||
"for",
|
"for",
|
||||||
"or",
|
"or",
|
||||||
"vs",
|
"vs",
|
||||||
"the",
|
|
||||||
"és",
|
"és",
|
||||||
"az",
|
"az",
|
||||||
"egy",
|
"egy",
|
||||||
@@ -120,7 +119,6 @@ _ACRONYMS = {
|
|||||||
"nfl",
|
"nfl",
|
||||||
"asmr",
|
"asmr",
|
||||||
"pov",
|
"pov",
|
||||||
"diy",
|
|
||||||
"wtf",
|
"wtf",
|
||||||
"lol",
|
"lol",
|
||||||
"rtx",
|
"rtx",
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import threading
|
|||||||
import time
|
import time
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from functools import partial
|
||||||
|
|
||||||
from sqlalchemy import text
|
from sqlalchemy import text
|
||||||
|
|
||||||
@@ -72,7 +73,7 @@ def _wait_for_schema(timeout: float = 300.0) -> bool:
|
|||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
|
db.execute(text("SELECT 1 FROM download_jobs LIMIT 1"))
|
||||||
return True
|
return True
|
||||||
except Exception: # noqa: BLE001 — table not migrated yet / DB not up yet
|
except Exception: # table not migrated yet / DB not up yet
|
||||||
if time.monotonic() > deadline:
|
if time.monotonic() > deadline:
|
||||||
# The traceback, not `str(exc)[:120]`: this is the last thing the worker says
|
# The traceback, not `str(exc)[:120]`: this is the last thing the worker says
|
||||||
# before it exits, and which driver call failed is the whole question.
|
# before it exits, and which driver call failed is the whole question.
|
||||||
@@ -366,7 +367,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
proc = _spawn_download_child(job_id)
|
proc = _spawn_download_child(job_id)
|
||||||
except Exception: # noqa: BLE001 — spawn failed; don't leave the asset stuck 'downloading'
|
except Exception: # spawn failed; don't leave the asset stuck 'downloading'
|
||||||
log.exception("failed to spawn download child for job %s", job_id)
|
log.exception("failed to spawn download child for job %s", job_id)
|
||||||
_reset_asset_pending(asset_id)
|
_reset_asset_pending(asset_id)
|
||||||
_requeue_if_running(job_id, None)
|
_requeue_if_running(job_id, None)
|
||||||
@@ -438,7 +439,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
|||||||
cause="child_exit",
|
cause="child_exit",
|
||||||
exit_code=proc.returncode,
|
exit_code=proc.returncode,
|
||||||
)
|
)
|
||||||
except Exception: # noqa: BLE001 — boot recovery is the backstop for a stuck 'running' job
|
except Exception: # boot recovery is the backstop for a stuck 'running' job
|
||||||
log.exception("post-exit reconciliation failed for job %s", job_id)
|
log.exception("post-exit reconciliation failed for job %s", job_id)
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -501,6 +502,7 @@ def _process_job(job_id: int) -> None:
|
|||||||
then run in a supervised CHILD PROCESS (`_run_supervised`) so a wedged yt-dlp/ffmpeg can be
|
then run in a supervised CHILD PROCESS (`_run_supervised`) so a wedged yt-dlp/ffmpeg can be
|
||||||
hard-killed — a worker THREAD cannot be killed, nor (when it spins in a C call) even
|
hard-killed — a worker THREAD cannot be killed, nor (when it spins in a C call) even
|
||||||
cooperatively cancelled, and one such thread starves the others via the GIL."""
|
cooperatively cancelled, and one such thread starves the others via the GIL."""
|
||||||
|
diagnosis = None # set exactly when the asset has already failed (asset_status == "error")
|
||||||
with SessionLocal() as db:
|
with SessionLocal() as db:
|
||||||
job = db.get(DownloadJob, job_id)
|
job = db.get(DownloadJob, job_id)
|
||||||
if job is None:
|
if job is None:
|
||||||
@@ -528,7 +530,7 @@ def _process_job(job_id: int) -> None:
|
|||||||
if asset_status == "ready":
|
if asset_status == "ready":
|
||||||
_finish_from_cache(job_id, asset_id)
|
_finish_from_cache(job_id, asset_id)
|
||||||
return
|
return
|
||||||
if asset_status == "error":
|
if diagnosis is not None:
|
||||||
_fail_running(
|
_fail_running(
|
||||||
job_id,
|
job_id,
|
||||||
error=asset_error or "Download failed",
|
error=asset_error or "Download failed",
|
||||||
@@ -577,7 +579,7 @@ def _worker_loop(idx: int) -> None:
|
|||||||
while not _stop.is_set():
|
while not _stop.is_set():
|
||||||
try:
|
try:
|
||||||
job_id = _claim_job()
|
job_id = _claim_job()
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
log.exception("claim failed")
|
log.exception("claim failed")
|
||||||
_stop.wait(settings.download_poll_seconds)
|
_stop.wait(settings.download_poll_seconds)
|
||||||
continue
|
continue
|
||||||
@@ -597,12 +599,12 @@ def _worker_loop(idx: int) -> None:
|
|||||||
job_id,
|
job_id,
|
||||||
exc,
|
exc,
|
||||||
"crash",
|
"crash",
|
||||||
lambda: _fail_running(job_id, error="Worker error", error_code=None),
|
partial(_fail_running, job_id, error="Worker error", error_code=None),
|
||||||
)
|
)
|
||||||
# silent-ok: not silent — `_report_worker_error` logged the crash before this
|
# silent-ok: not silent — `_report_worker_error` logged the crash before this
|
||||||
# re-raise. Failing the job hit the database, and letting that out would end this
|
# re-raise. Failing the job hit the database, and letting that out would end this
|
||||||
# worker thread for good; boot recovery settles a job left `running`.
|
# worker thread for good; boot recovery settles a job left `running`.
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
log.exception(
|
log.exception(
|
||||||
"could not fail job %s after its crash",
|
"could not fail job %s after its crash",
|
||||||
job_id,
|
job_id,
|
||||||
|
|||||||
@@ -114,18 +114,32 @@ _DEAD_GRANT = "invalid_grant"
|
|||||||
|
|
||||||
# What a refused YouTube action means to the person who asked for it — the three things they can
|
# What a refused YouTube action means to the person who asked for it — the three things they can
|
||||||
# act on differently: wait for tomorrow's budget, reconnect the account, or try again later.
|
# act on differently: wait for tomorrow's budget, reconnect the account, or try again later.
|
||||||
YouTubeRefusal = Literal["quota", "reconnect", "unavailable"]
|
YouTubeRefusal = Literal["quota", "reconnect", "unavailable", "gone"]
|
||||||
|
|
||||||
|
|
||||||
def refusal_reason(exc: YouTubeError) -> YouTubeRefusal:
|
def is_playlist_gone(exc: YouTubeError) -> bool:
|
||||||
|
"""Whether YouTube answered that the playlist read or written does not exist. Only the caller
|
||||||
|
knows which playlist that was: a user's list deleted on YouTube, or a channel's uploads list,
|
||||||
|
which YouTube also answers so for a channel with no uploads at all."""
|
||||||
|
return "playlistNotFound" in exc.reason.split(",")
|
||||||
|
|
||||||
|
|
||||||
|
def refusal_reason(exc: YouTubeError, *, playlist: bool = False) -> YouTubeRefusal:
|
||||||
"""Sort a failure by what the user can do about it. A dead grant (Google refused the refresh
|
"""Sort a failure by what the user can do about it. A dead grant (Google refused the refresh
|
||||||
with `invalid_grant`, which also drops the grant — see `_drop_dead_grant`) or no usable token at
|
with `invalid_grant`, which also drops the grant — see `_drop_dead_grant`) or no usable token at
|
||||||
all (raised before any endpoint answered, so with no path and no transport reason) means the
|
all (raised before any endpoint answered, so with no path and no transport reason) means the
|
||||||
account must be reconnected; retrying cannot help. Any other token-endpoint failure — a 5xx, a
|
account must be reconnected; retrying cannot help. Any other token-endpoint failure — a 5xx, a
|
||||||
429, a misconfigured client — leaves the grant as it was: reconnecting would not fix it, and
|
429, a misconfigured client — leaves the grant as it was: reconnecting would not fix it, and
|
||||||
the account would still read as connected, with no connect button to press."""
|
the account would still read as connected, with no connect button to press.
|
||||||
|
|
||||||
|
`playlist`: the action was on ONE playlist the user picked, so a missing playlist is that list,
|
||||||
|
gone (deleted on YouTube) — neither a retry nor a reconnect helps. Anywhere else the playlist
|
||||||
|
is one the user never named (a channel's uploads, one list of a bulk pull), and "this playlist
|
||||||
|
no longer exists" would describe nothing on their screen (E32 review round 2)."""
|
||||||
if is_quota_refusal(exc):
|
if is_quota_refusal(exc):
|
||||||
return "quota"
|
return "quota"
|
||||||
|
if playlist and is_playlist_gone(exc):
|
||||||
|
return "gone"
|
||||||
if exc.path == "oauth2/token":
|
if exc.path == "oauth2/token":
|
||||||
return "reconnect" if exc.reason == _DEAD_GRANT else "unavailable"
|
return "reconnect" if exc.reason == _DEAD_GRANT else "unavailable"
|
||||||
if not exc.path and exc.reason != "transport":
|
if not exc.path and exc.reason != "transport":
|
||||||
@@ -189,7 +203,7 @@ class YouTubeClient:
|
|||||||
return cached
|
return cached
|
||||||
try:
|
try:
|
||||||
refresh = decrypt(tok.refresh_token_enc)
|
refresh = decrypt(tok.refresh_token_enc)
|
||||||
except Exception as exc: # noqa: BLE001 — Fernet InvalidToken (TOKEN_ENCRYPTION_KEY changed)
|
except Exception as exc: # Fernet InvalidToken (TOKEN_ENCRYPTION_KEY changed)
|
||||||
# A key mismatch must degrade like any other token problem (caller handles YouTubeError),
|
# A key mismatch must degrade like any other token problem (caller handles YouTubeError),
|
||||||
# not surface as a raw 500 on every scheduler job / connect check. (B-core-2)
|
# not surface as a raw 500 on every scheduler job / connect check. (B-core-2)
|
||||||
raise YouTubeError("Stored OAuth token is unreadable — reconnect your account") from exc
|
raise YouTubeError("Stored OAuth token is unreadable — reconnect your account") from exc
|
||||||
@@ -423,16 +437,22 @@ class YouTubeClient:
|
|||||||
return resp.json() if resp.content else {}
|
return resp.json() if resp.content else {}
|
||||||
|
|
||||||
def get_playlist_snippet(self, playlist_id: str) -> dict | None:
|
def get_playlist_snippet(self, playlist_id: str) -> dict | None:
|
||||||
"""The snippet (title/description) of one playlist, or None if it no longer
|
"""The snippet (title/description) of one playlist plus its `item_count` (what an empty
|
||||||
exists. 1 unit (OAuth, so private playlists work)."""
|
item listing is checked against), or None if it no longer exists. 1 unit (OAuth, so
|
||||||
|
private playlists work)."""
|
||||||
data = self._get(
|
data = self._get(
|
||||||
"playlists",
|
"playlists",
|
||||||
{"part": "snippet", "id": playlist_id, "maxResults": 1},
|
{"part": "snippet,contentDetails", "id": playlist_id, "maxResults": 1},
|
||||||
cost=1,
|
cost=1,
|
||||||
allow_key=False,
|
allow_key=False,
|
||||||
)
|
)
|
||||||
items = data.get("items", [])
|
items = data.get("items", [])
|
||||||
return items[0].get("snippet") if items else None
|
if not items:
|
||||||
|
return None
|
||||||
|
return {
|
||||||
|
**(items[0].get("snippet") or {}),
|
||||||
|
"item_count": items[0].get("contentDetails", {}).get("itemCount"),
|
||||||
|
}
|
||||||
|
|
||||||
def update_playlist_title(self, playlist_id: str, title: str) -> None:
|
def update_playlist_title(self, playlist_id: str, title: str) -> None:
|
||||||
"""Rename a playlist on YouTube. 50 units."""
|
"""Rename a playlist on YouTube. 50 units."""
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ class RssError(Exception):
|
|||||||
def _get_feed(channel_id: str) -> bytes:
|
def _get_feed(channel_id: str) -> bytes:
|
||||||
"""The feed's body, after up to `1 + len(RETRY_DELAYS)` tries; the LAST failure is raised."""
|
"""The feed's body, after up to `1 + len(RETRY_DELAYS)` tries; the LAST failure is raised."""
|
||||||
url = RSS_URL.format(channel_id=channel_id)
|
url = RSS_URL.format(channel_id=channel_id)
|
||||||
|
error: RssError | None = None
|
||||||
for attempt in range(len(RETRY_DELAYS) + 1):
|
for attempt in range(len(RETRY_DELAYS) + 1):
|
||||||
if attempt:
|
if attempt:
|
||||||
time.sleep(RETRY_DELAYS[attempt - 1])
|
time.sleep(RETRY_DELAYS[attempt - 1])
|
||||||
@@ -60,6 +61,8 @@ def _get_feed(channel_id: str) -> bytes:
|
|||||||
)
|
)
|
||||||
if resp.status_code not in _RETRYABLE_STATUS:
|
if resp.status_code not in _RETRYABLE_STATUS:
|
||||||
break
|
break
|
||||||
|
if error is None: # cannot happen: the range above is never empty, and every try records one
|
||||||
|
raise RssError(f"RSS fetch for {channel_id} made no attempt", "transport")
|
||||||
raise error
|
raise error
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -42,6 +42,9 @@ _HEADERS = {
|
|||||||
"Accept-Language": "en-US,en;q=0.9",
|
"Accept-Language": "en-US,en;q=0.9",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# The top-level keys a first page (contents) and a continuation page (onResponseReceivedCommands)
|
||||||
|
# put their result list under.
|
||||||
|
_RESULT_FRAMES = {"contents", "onResponseReceivedCommands"}
|
||||||
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
|
_KEY_RE = re.compile(r'"INNERTUBE_API_KEY":"([^"]+)"')
|
||||||
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
|
_VER_RE = re.compile(r'"INNERTUBE_CLIENT_VERSION":"([^"]+)"')
|
||||||
|
|
||||||
@@ -65,29 +68,36 @@ def _runs_text(node: dict | None) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def _get_config(client: httpx.Client) -> tuple[str, str]:
|
def _get_config(client: httpx.Client) -> tuple[str, str]:
|
||||||
"""The current InnerTube (api_key, client_version), cached after the first fetch."""
|
"""The current InnerTube (api_key, client_version), cached after the first fetch that FOUND them.
|
||||||
|
|
||||||
|
The fallback pair is never cached: it used to be, so one failed fetch (or a consent page with no
|
||||||
|
keys on it) at the wrong moment pinned every later search to a hard-coded key and version for
|
||||||
|
the life of the process — and when those went stale, search returned nothing, with no error
|
||||||
|
anywhere (E32 S2). Now the next search simply tries the live pair again."""
|
||||||
global _config
|
global _config
|
||||||
if _config is not None:
|
if _config is not None:
|
||||||
return _config
|
return _config
|
||||||
try:
|
try:
|
||||||
resp = client.get(_RESULTS_URL, params={"search_query": "music"})
|
html = client.get(_RESULTS_URL, params={"search_query": "music"}).text
|
||||||
html = resp.text
|
|
||||||
key = _KEY_RE.search(html)
|
|
||||||
ver = _VER_RE.search(html)
|
|
||||||
_config = (
|
|
||||||
key.group(1) if key else _FALLBACK_KEY,
|
|
||||||
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
|
|
||||||
)
|
|
||||||
except httpx.HTTPError as exc:
|
except httpx.HTTPError as exc:
|
||||||
# The fallback pair is CACHED in `_config` for the life of the process, so one failed
|
|
||||||
# fetch at the wrong moment pins every later search to a hard-coded key and version —
|
|
||||||
# and when those go stale, search returns nothing at all, with no error anywhere.
|
|
||||||
log.warning(
|
log.warning(
|
||||||
"the InnerTube config could not be fetched; falling back to the pinned key",
|
"the InnerTube config could not be fetched; falling back to the pinned key",
|
||||||
extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
|
extra=logs.event("youtube.scrape.config_fallback", reason=type(exc).__name__),
|
||||||
)
|
)
|
||||||
_config = (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
|
return (_FALLBACK_KEY, _FALLBACK_CLIENT_VERSION)
|
||||||
return _config
|
key = _KEY_RE.search(html)
|
||||||
|
ver = _VER_RE.search(html)
|
||||||
|
if key and ver:
|
||||||
|
_config = (key.group(1), ver.group(1))
|
||||||
|
return _config
|
||||||
|
log.warning(
|
||||||
|
"the results page carried no InnerTube config; falling back to the pinned key",
|
||||||
|
extra=logs.event("youtube.scrape.config_fallback", reason="not_on_page"),
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
key.group(1) if key else _FALLBACK_KEY,
|
||||||
|
ver.group(1) if ver else _FALLBACK_CLIENT_VERSION,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _video_item(vr: dict) -> dict | None:
|
def _video_item(vr: dict) -> dict | None:
|
||||||
@@ -182,6 +192,10 @@ def search(q: str, continuation: str | None = None) -> dict:
|
|||||||
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
|
except ValueError as exc: # non-JSON body (blocked / consent / captcha)
|
||||||
raise ScrapeError("InnerTube returned a non-JSON response") from exc
|
raise ScrapeError("InnerTube returned a non-JSON response") from exc
|
||||||
|
|
||||||
|
# Even a search with no results carries its result list's frame. Without one this is not "no
|
||||||
|
# results" but a shape we do not know (YouTube moved, or a block page in JSON) — say so.
|
||||||
|
if not isinstance(data, dict) or not _RESULT_FRAMES & data.keys():
|
||||||
|
raise ScrapeError("InnerTube returned an unexpected shape")
|
||||||
items: list[dict] = []
|
items: list[dict] = []
|
||||||
_collect_videos(data, items, set())
|
_collect_videos(data, items, set())
|
||||||
return {"items": items, "next_page_token": _find_continuation(data)}
|
return {"items": items, "next_page_token": _find_continuation(data)}
|
||||||
|
|||||||
@@ -35,4 +35,9 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
- ./:/app
|
- ./:/app
|
||||||
working_dir: /app
|
working_dir: /app
|
||||||
command: python -m pytest
|
# The suite, then the coverage total (E32 D4: reported, no threshold). pytest's exit code is the
|
||||||
|
# lane's — the report is printed either way and never decides it.
|
||||||
|
command:
|
||||||
|
- sh
|
||||||
|
- -c
|
||||||
|
- 'python -m pytest --cov=app --cov-report= ; rc=$$?; echo ">> backend coverage (app/) $$(python -m coverage report --format=total)% of lines"; exit $$rc'
|
||||||
|
|||||||
@@ -25,7 +25,7 @@
|
|||||||
{
|
{
|
||||||
"include": ["app", "alembic"],
|
"include": ["app", "alembic"],
|
||||||
"extraPaths": ["."],
|
"extraPaths": ["."],
|
||||||
"typeCheckingMode": "basic",
|
"typeCheckingMode": "standard",
|
||||||
"pythonVersion": "3.12",
|
"pythonVersion": "3.12",
|
||||||
"pythonPlatform": "Linux"
|
"pythonPlatform": "Linux"
|
||||||
}
|
}
|
||||||
|
|||||||
+4
-2
@@ -2,7 +2,9 @@
|
|||||||
# Run from backend/ (the image's /app). `pythonpath = .` puts the app package on sys.path so tests
|
# Run from backend/ (the image's /app). `pythonpath = .` puts the app package on sys.path so tests
|
||||||
# import it as `app.*`, matching runtime. Most of the suite is pure-logic; the DB-backed lane opts in
|
# import it as `app.*`, matching runtime. Most of the suite is pure-logic; the DB-backed lane opts in
|
||||||
# via the `db` fixture and runs in the compose test image. `-rs` reports skip REASONS so a silently
|
# via the `db` fixture and runs in the compose test image. `-rs` reports skip REASONS so a silently
|
||||||
# skipped lane is visible (the compose lane sets SIFTLODE_TEST_REQUIRE_DB=1 to make skips a hard error).
|
# skipped lane is visible (and conftest makes a missing DB/ffmpeg a hard error unless
|
||||||
|
# SIFTLODE_TEST_REQUIRE_DB=0). `--strict-markers` (E32): a misspelt `@pytest.mark.skipiff` is an error,
|
||||||
|
# not an unknown marker that quietly does nothing.
|
||||||
testpaths = tests
|
testpaths = tests
|
||||||
pythonpath = .
|
pythonpath = .
|
||||||
addopts = -q -rs
|
addopts = -q -rs --strict-markers
|
||||||
|
|||||||
@@ -7,7 +7,10 @@
|
|||||||
# basedpyright: the backend type-check lane (E27 S3). Pinned like everything else here — a type
|
# basedpyright: the backend type-check lane (E27 S3). Pinned like everything else here — a type
|
||||||
# checker whose version floats silently changes what "clean" means, and the ratchet baseline it
|
# checker whose version floats silently changes what "clean" means, and the ratchet baseline it
|
||||||
# writes is keyed to the diagnostics THIS version produces.
|
# writes is keyed to the diagnostics THIS version produces.
|
||||||
|
# pytest-cov: the coverage REPORT the gate prints (E32 D4) — a number to read, deliberately with no
|
||||||
|
# threshold; "every regression test seen red" is the bar, not a percentage.
|
||||||
basedpyright==1.40.1
|
basedpyright==1.40.1
|
||||||
pytest==8.4.2
|
pytest==8.4.2
|
||||||
|
pytest-cov==7.1.0
|
||||||
ruff==0.15.21
|
ruff==0.15.21
|
||||||
httpx2==2.13.0
|
httpx2==2.13.0
|
||||||
|
|||||||
@@ -1,7 +1,10 @@
|
|||||||
# Pinned dependency lock for reproducible builds (SEC-5). GENERATED — do not hand-edit.
|
# Pinned dependency lock for reproducible builds (SEC-5). GENERATED — do not hand-edit.
|
||||||
# Source of truth is requirements.txt (the floor ranges); regenerate after editing it with:
|
# Source of truth is requirements.txt (the floor ranges); regenerate after editing it with:
|
||||||
# docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort
|
# docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort
|
||||||
# The prod Dockerfile + Dockerfile.test + CI install THIS file; pip-audit scans it in CI.
|
# The prod Dockerfile + Dockerfile.test + CI install THIS file; pip-audit scans it in CI (blocking
|
||||||
|
# since E32). A security fix may be ONE pin bumped in place instead of a full regeneration, which
|
||||||
|
# would move every other package too — then `pip install -r requirements.lock && pip check` must
|
||||||
|
# pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177, E32).
|
||||||
APScheduler==3.11.3
|
APScheduler==3.11.3
|
||||||
Authlib==1.7.2
|
Authlib==1.7.2
|
||||||
Mako==1.4.1
|
Mako==1.4.1
|
||||||
@@ -53,7 +56,7 @@ starlette==1.6.0
|
|||||||
typing-inspection==0.4.4
|
typing-inspection==0.4.4
|
||||||
typing_extensions==4.16.0
|
typing_extensions==4.16.0
|
||||||
tzlocal==5.4.4
|
tzlocal==5.4.4
|
||||||
urllib3==2.7.0
|
urllib3==2.8.0
|
||||||
uvicorn==0.52.3
|
uvicorn==0.52.3
|
||||||
uvloop==0.22.1
|
uvloop==0.22.1
|
||||||
watchfiles==1.2.0
|
watchfiles==1.2.0
|
||||||
|
|||||||
+27
-8
@@ -1,7 +1,8 @@
|
|||||||
# Ruff config for the Siftlode backend. Keeps ruff's default rule set (E4/E7/E9 + F = the
|
# Ruff config for the Siftlode backend. Keeps ruff's default rule set (E4/E7/E9 + F = the
|
||||||
# high-signal pyflakes/pycodestyle checks that catch real dead code + bugs), but ignores E402:
|
# high-signal pyflakes/pycodestyle checks that catch real dead code + bugs), all of it on. E402 was
|
||||||
# a handful of modules (e.g. main.py) deliberately run setup — logging config, sys.path — before
|
# ignored for main.py's logging setup; the ignore had let 11 incidental late imports in elsewhere
|
||||||
# importing route modules, so "module import not at top of file" is intentional there, not cruft.
|
# (E32 review), so it is on now and main.py marks each deliberate one (`# noqa: E402 — <why>`).
|
||||||
|
# `scripts/e2e_seed.py` edits `sys.path` before its imports, which E402 itself allows.
|
||||||
#
|
#
|
||||||
# line-length 100 (ruff's default is 88): matches the frontend prettier printWidth so both halves of
|
# line-length 100 (ruff's default is 88): matches the frontend prettier printWidth so both halves of
|
||||||
# the repo wrap at the same width, and it's the width `ruff format` enforces (gated via
|
# the repo wrap at the same width, and it's the width `ruff format` enforces (gated via
|
||||||
@@ -17,9 +18,10 @@ target-version = "py312"
|
|||||||
# LOG — stdlib logging misuse (`.warn()`, a logger built by hand, `exc_info` on `.exception`)
|
# LOG — stdlib logging misuse (`.warn()`, a logger built by hand, `exc_info` on `.exception`)
|
||||||
# G — the call's shape: %-style args, never an f-string the level filter cannot skip, and
|
# G — the call's shape: %-style args, never an f-string the level filter cannot skip, and
|
||||||
# never `str(exc)` where `exc_info` would carry the traceback
|
# never `str(exc)` where `exc_info` would carry the traceback
|
||||||
# T20 — `print` instead of a log line. Two deliberate ones remain, each with its `# noqa`
|
# T20 — `print` instead of a log line. Three deliberate ones remain, each with its `# noqa`
|
||||||
# and a why: `downloads/runner.py`'s usage line is for whoever typed it, and
|
# and a why: `downloads/runner.py`'s usage line is for whoever typed it, `logsink.py`
|
||||||
# `logsink.py` reports the SINK's own failure, which cannot be logged into the sink.
|
# reports the SINK's own failure, which cannot be logged into the sink, and
|
||||||
|
# `scripts/e2e_seed.py` prints the seeded ids for the e2e run that reads them.
|
||||||
# TRY400 — `.error()` in an `except` where `.exception()` would keep the traceback
|
# TRY400 — `.error()` in an `except` where `.exception()` would keep the traceback
|
||||||
# TRY401 — the exception spelled into the message AND passed as `exc_info` (it is in both)
|
# TRY401 — the exception spelled into the message AND passed as `exc_info` (it is in both)
|
||||||
# S112 — `except: continue`, the silent swallow inside a loop
|
# S112 — `except: continue`, the silent swallow inside a loop
|
||||||
@@ -29,5 +31,22 @@ target-version = "py312"
|
|||||||
# nobody to raise to, a best-effort backfill inside a sync) and always worth a sentence.
|
# nobody to raise to, a best-effort backfill inside a sync) and always worth a sentence.
|
||||||
# The wider rule these two enforce lives in tests/test_swallowed_exceptions.py: a handler that
|
# The wider rule these two enforce lives in tests/test_swallowed_exceptions.py: a handler that
|
||||||
# neither logs nor re-raises says `# silent-ok: <why>`. Ruff sees only the two narrow shapes.
|
# neither logs nor re-raises says `# silent-ok: <why>`. Ruff sees only the two narrow shapes.
|
||||||
extend-select = ["LOG", "G", "T20", "TRY400", "TRY401", "S110", "S112", "BLE001"]
|
#
|
||||||
ignore = ["E402"]
|
# E32 (gate integrity) added three more, each measured before it went on and with every hit fixed:
|
||||||
|
# RUF100 — a `# noqa` that suppresses nothing. A stale one is a claim with no rule behind it, and
|
||||||
|
# it stays armed for the day the line changes (24 were removed). A comment that merely
|
||||||
|
# explains a noqa below it names the rule as `# TRY400: …`, never `# noqa TRY400`.
|
||||||
|
# PLW1510 — `subprocess.run` without an explicit `check=`. Found the real E32 bug: three ffprobe
|
||||||
|
# probes read the output of a probe that had FAILED and took it for the file's truth.
|
||||||
|
# B — flake8-bugbear: loop-variable capture, `raise` without `from` inside `except`, a
|
||||||
|
# `zip` that silently truncates (each one is `strict=True` now — the lengths are an
|
||||||
|
# invariant, so a mismatch should be loud). B008 (a call in a default argument) stays ON:
|
||||||
|
# FastAPI's `Depends(...)` / `Query(...)` ARE that shape by design (474 hits, all of
|
||||||
|
# them), so those two are declared immutable below instead of switching the rule off —
|
||||||
|
# any other call in a default still fails.
|
||||||
|
extend-select = [
|
||||||
|
"LOG", "G", "T20", "TRY400", "TRY401", "S110", "S112", "BLE001", "RUF100", "PLW1510", "B",
|
||||||
|
]
|
||||||
|
|
||||||
|
[lint.flake8-bugbear]
|
||||||
|
extend-immutable-calls = ["fastapi.Depends", "fastapi.Query"]
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ from fastapi.testclient import TestClient
|
|||||||
from sqlalchemy import create_engine, event, text
|
from sqlalchemy import create_engine, event, text
|
||||||
from starlette.requests import HTTPConnection
|
from starlette.requests import HTTPConnection
|
||||||
|
|
||||||
import app.models # noqa: F401 — register every model on Base.metadata
|
import app.models # register every model on Base.metadata
|
||||||
import app.state as app_state
|
import app.state as app_state
|
||||||
from app import security
|
from app import security
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
@@ -37,9 +37,11 @@ from app.models import User
|
|||||||
# Sentinel for "this dependency had no override before the fixture set one" (see override_dep).
|
# Sentinel for "this dependency had no override before the fixture set one" (see override_dep).
|
||||||
_MISSING = object()
|
_MISSING = object()
|
||||||
|
|
||||||
# The compose test lane sets this so a would-be silent skip (no DB, or no ffmpeg) becomes a hard error
|
# A would-be silent skip (no DB, or no ffmpeg) is a hard error unless the run says otherwise — the
|
||||||
# — the gate must not print green while the DB-backed / ffmpeg tests were quietly not run.
|
# suite must not print green while the DB-backed / ffmpeg tests were quietly not run. ON by default
|
||||||
_REQUIRE_FULL = os.getenv("SIFTLODE_TEST_REQUIRE_DB") == "1"
|
# since E32: as an opt-in only the compose lane and CI set, a hand-run `pytest` skipped ~60 DB tests
|
||||||
|
# and still passed. `SIFTLODE_TEST_REQUIRE_DB=0` is the explicit "pure-logic only" run.
|
||||||
|
_REQUIRE_FULL = os.getenv("SIFTLODE_TEST_REQUIRE_DB", "1") != "0"
|
||||||
|
|
||||||
|
|
||||||
def _db_reachable() -> bool:
|
def _db_reachable() -> bool:
|
||||||
@@ -86,19 +88,20 @@ def _truncate_all():
|
|||||||
|
|
||||||
|
|
||||||
def pytest_configure(config):
|
def pytest_configure(config):
|
||||||
"""Fail the whole run — loudly, once — when SIFTLODE_TEST_REQUIRE_DB=1 but the environment can't
|
"""Fail the whole run — loudly, once — when the full suite is required (the default) but the
|
||||||
actually run the full suite. Without this, an unreachable DATABASE_URL or a missing ffmpeg turns
|
environment can't actually run it. Without this, an unreachable DATABASE_URL or a missing ffmpeg turns
|
||||||
~60 DB tests + the poster tests into silent skips and the gate still exits green."""
|
~60 DB tests + the poster tests into silent skips and the gate still exits green."""
|
||||||
if not _REQUIRE_FULL:
|
if not _REQUIRE_FULL:
|
||||||
return
|
return
|
||||||
if not _DB_REACHABLE:
|
if not _DB_REACHABLE:
|
||||||
raise pytest.UsageError(
|
raise pytest.UsageError(
|
||||||
"SIFTLODE_TEST_REQUIRE_DB=1 but no Postgres is reachable at the test DATABASE_URL — the "
|
"no Postgres is reachable at the test DATABASE_URL — the DB-backed lane would silently "
|
||||||
"DB-backed lane would silently skip. Fix the DB wiring or unset the flag."
|
"skip. Fix the DB wiring, or run the pure-logic tests only with SIFTLODE_TEST_REQUIRE_DB=0."
|
||||||
)
|
)
|
||||||
if shutil.which("ffmpeg") is None:
|
if shutil.which("ffmpeg") is None:
|
||||||
raise pytest.UsageError(
|
raise pytest.UsageError(
|
||||||
"SIFTLODE_TEST_REQUIRE_DB=1 but ffmpeg is missing — the thumbnail/poster tests would skip."
|
"ffmpeg is missing — the thumbnail/poster tests would skip. Install it, or run the "
|
||||||
|
"pure-logic tests only with SIFTLODE_TEST_REQUIRE_DB=0."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -36,9 +36,14 @@ def test_a_public_url_gets_past_the_guard(db, as_user, make_user, monkeypatch):
|
|||||||
"getaddrinfo",
|
"getaddrinfo",
|
||||||
lambda *a, **kw: [(None, None, None, "", ("93.184.216.34", 0))],
|
lambda *a, **kw: [(None, None, None, "", ("93.184.216.34", 0))],
|
||||||
)
|
)
|
||||||
r = as_user(user).post("/api/downloads", json={"source": "https://example.com/v"})
|
# An inline spec, so nothing ELSE refuses it: without one the request stopped at the
|
||||||
detail = r.json().get("detail") if r.status_code >= 400 else None
|
# profile lookup (400 no_profile) and "not url_blocked" held for a request the guard never
|
||||||
assert not (isinstance(detail, dict) and detail.get("code") == "url_blocked")
|
# had to wave through (E32 S2).
|
||||||
|
r = as_user(user).post(
|
||||||
|
"/api/downloads", json={"source": "https://example.com/v", "spec": {"format": "best"}}
|
||||||
|
)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
assert r.json()["status"] == "queued"
|
||||||
|
|
||||||
|
|
||||||
def test_youtube_sources_never_consult_the_guard(db, as_user, make_user, monkeypatch):
|
def test_youtube_sources_never_consult_the_guard(db, as_user, make_user, monkeypatch):
|
||||||
@@ -50,7 +55,13 @@ def test_youtube_sources_never_consult_the_guard(db, as_user, make_user, monkeyp
|
|||||||
user = make_user()
|
user = make_user()
|
||||||
db.commit()
|
db.commit()
|
||||||
monkeypatch.setattr(urlguard.socket, "getaddrinfo", boom)
|
monkeypatch.setattr(urlguard.socket, "getaddrinfo", boom)
|
||||||
as_user(user).post("/api/downloads", json={"source": "dQw4w9WgXcQ"})
|
# The job must actually be created: a request refused for any other reason would never reach
|
||||||
|
# the guard either, and the test would pass without proving anything (E32 S2).
|
||||||
|
r = as_user(user).post(
|
||||||
|
"/api/downloads", json={"source": "dQw4w9WgXcQ", "spec": {"format": "best"}}
|
||||||
|
)
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
assert r.json()["source_kind"] == "youtube"
|
||||||
|
|
||||||
|
|
||||||
def test_manual_subscription_sync_goes_on_cooldown(db, as_user, make_user, monkeypatch):
|
def test_manual_subscription_sync_goes_on_cooldown(db, as_user, make_user, monkeypatch):
|
||||||
@@ -81,8 +92,12 @@ def test_the_cooldown_is_per_user_not_global(db, as_user, make_user, monkeypatch
|
|||||||
monkeypatch.setattr(syncroutes, "import_subscriptions", lambda d, u: {"imported": 0})
|
monkeypatch.setattr(syncroutes, "import_subscriptions", lambda d, u: {"imported": 0})
|
||||||
|
|
||||||
ca = as_user(a)
|
ca = as_user(a)
|
||||||
for _ in range(4):
|
assert [ca.post("/api/sync/subscriptions").status_code for _ in range(4)] == [
|
||||||
ca.post("/api/sync/subscriptions")
|
200,
|
||||||
|
200,
|
||||||
|
200,
|
||||||
|
429,
|
||||||
|
]
|
||||||
assert ca.post("/api/sync/subscriptions").status_code == 429
|
assert ca.post("/api/sync/subscriptions").status_code == 429
|
||||||
assert as_user(b).post("/api/sync/subscriptions").status_code == 200
|
assert as_user(b).post("/api/sync/subscriptions").status_code == 200
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""sync/autotag's orphan cleanup (E32 S2: the module was 32% covered).
|
||||||
|
|
||||||
|
A system tag no channel carries any more is deleted — unless a saved feed view filters on it. The
|
||||||
|
views hold tag ids; a deleted tag returns under a NEW id when a channel earns it again, and the feed
|
||||||
|
drops an unknown id, so the view silently showed the whole feed under its old name.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.models import Channel, ChannelTag, SavedView, Tag
|
||||||
|
from app.sync import autotag
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_orphan_system_tag_goes_but_one_a_saved_view_filters_on_stays(db, make_user):
|
||||||
|
user = make_user()
|
||||||
|
used = Tag(user_id=None, name="Gaming", category="topic")
|
||||||
|
orphan = Tag(user_id=None, name="Cooking", category="topic")
|
||||||
|
in_view = Tag(user_id=None, name="Music", category="topic")
|
||||||
|
own = Tag(user_id=user.id, name="Mine", category="other")
|
||||||
|
ch = Channel(id="UC_tags", title="T")
|
||||||
|
db.add_all([used, orphan, in_view, own, ch])
|
||||||
|
db.flush()
|
||||||
|
db.add(ChannelTag(channel_id=ch.id, tag_id=used.id, user_id=None, source="auto"))
|
||||||
|
db.add(
|
||||||
|
SavedView(
|
||||||
|
user_id=user.id, source="feed", name="Music", filters={"tags": [in_view.id], "q": ""}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# A Plex view's filters are another vocabulary; they must not break the scan.
|
||||||
|
db.add(SavedView(user_id=user.id, source="plex", name="P", filters={"genres": ["Drama"]}))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
assert autotag._cleanup_orphan_system_tags(db) == 1
|
||||||
|
db.expire_all()
|
||||||
|
names = set(db.execute(select(Tag.name)).scalars())
|
||||||
|
assert names == {"Gaming", "Music", "Mine"} # only the unreferenced system orphan went
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_malformed_saved_view_cannot_break_the_cleanup(db, make_user):
|
||||||
|
"""A view stored around the route's shape check (or before it) must not raise for everyone's job
|
||||||
|
— `"tags": 5` raised TypeError — and `True` is not tag id 1 (E32 review)."""
|
||||||
|
user = make_user()
|
||||||
|
orphan = Tag(user_id=None, name="Cooking", category="topic")
|
||||||
|
db.add(orphan)
|
||||||
|
db.flush()
|
||||||
|
for i, filters in enumerate(
|
||||||
|
({"tags": 5}, {"tags": [True]}, ["not", "a", "dict"], {"tags": None})
|
||||||
|
):
|
||||||
|
db.add(SavedView(user_id=user.id, source="feed", name=f"bad{i}", filters=filters))
|
||||||
|
db.commit()
|
||||||
|
assert autotag._cleanup_orphan_system_tags(db) == 1
|
||||||
|
db.expire_all()
|
||||||
|
assert db.execute(select(Tag).where(Tag.name == "Cooking")).first() is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_view_kept_tag_list_larger_than_a_bind_parameter_limit_still_cleans_up(db, make_user):
|
||||||
|
user = make_user()
|
||||||
|
orphan = Tag(user_id=None, name="Cooking", category="topic")
|
||||||
|
db.add(orphan)
|
||||||
|
db.flush()
|
||||||
|
huge = list(range(10_000_000, 10_070_000)) # past psycopg's 65535 bind parameters
|
||||||
|
db.add(SavedView(user_id=user.id, source="feed", name="huge", filters={"tags": huge}))
|
||||||
|
db.commit()
|
||||||
|
assert autotag._cleanup_orphan_system_tags(db) == 1
|
||||||
@@ -321,7 +321,7 @@ def _body_reading_functions() -> dict[tuple[str, str], dict]:
|
|||||||
continue
|
continue
|
||||||
names = [name for _, name in calls]
|
names = [name for _, name in calls]
|
||||||
limits = []
|
limits = []
|
||||||
for node, name in zip([n for n, _ in calls], names):
|
for node, name in zip([n for n, _ in calls], names, strict=True):
|
||||||
if name.startswith("request."):
|
if name.startswith("request."):
|
||||||
limits.append("<no ceiling declared>")
|
limits.append("<no ceiling declared>")
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -151,7 +151,7 @@ def test_nobody_reaches_another_users_session(db, make_user, as_user):
|
|||||||
def test_the_list_is_the_users_own_newest_first(db, make_user, as_user):
|
def test_the_list_is_the_users_own_newest_first(db, make_user, as_user):
|
||||||
client = as_user(make_user())
|
client = as_user(make_user())
|
||||||
first = _start(client)["id"]
|
first = _start(client)["id"]
|
||||||
client.post(f"/api/debug-sessions/{first}/stop")
|
assert client.post(f"/api/debug-sessions/{first}/stop").status_code == 200
|
||||||
second = _start(client)["id"]
|
second = _start(client)["id"]
|
||||||
assert [s["id"] for s in client.get("/api/debug-sessions").json()["items"]] == [second, first]
|
assert [s["id"] for s in client.get("/api/debug-sessions").json()["items"]] == [second, first]
|
||||||
|
|
||||||
@@ -199,10 +199,13 @@ def test_an_erased_account_takes_its_sessions_with_it(db, make_user, as_user):
|
|||||||
user = make_user()
|
user = make_user()
|
||||||
client = as_user(user)
|
client = as_user(user)
|
||||||
sid = _start(client)["id"]
|
sid = _start(client)["id"]
|
||||||
client.post(
|
assert (
|
||||||
f"/api/debug-sessions/{sid}/screenshots",
|
client.post(
|
||||||
content=_PNG,
|
f"/api/debug-sessions/{sid}/screenshots",
|
||||||
headers={"content-type": "image/png"},
|
content=_PNG,
|
||||||
|
headers={"content-type": "image/png"},
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
purge_user(db, user, BackgroundTasks())
|
purge_user(db, user, BackgroundTasks())
|
||||||
assert db.get(DebugSession, sid) is None
|
assert db.get(DebugSession, sid) is None
|
||||||
@@ -250,7 +253,7 @@ def test_a_request_naming_the_own_open_session_is_tagged_and_raised(gated, db, c
|
|||||||
_signed_in(client, make_user)
|
_signed_in(client, make_user)
|
||||||
sid = _start(client, "DEBUG")["id"]
|
sid = _start(client, "DEBUG")["id"]
|
||||||
gated.records.clear()
|
gated.records.clear()
|
||||||
client.get("/api/debug-sessions", headers={"x-debug-session": sid})
|
assert client.get("/api/debug-sessions", headers={"x-debug-session": sid}).status_code == 200
|
||||||
(line,) = _request_lines(gated)
|
(line,) = _request_lines(gated)
|
||||||
assert getattr(line, "debug_session_id", None) == sid
|
assert getattr(line, "debug_session_id", None) == sid
|
||||||
# Raised to the session's level: the line is kept for storage though the user's own is Alap.
|
# Raised to the session's level: the line is kept for storage though the user's own is Alap.
|
||||||
@@ -263,7 +266,7 @@ def test_a_header_naming_anything_else_is_ignored_not_refused(gated, db, client,
|
|||||||
db.commit()
|
db.commit()
|
||||||
_signed_in(client, make_user)
|
_signed_in(client, make_user)
|
||||||
own = _start(client)["id"]
|
own = _start(client)["id"]
|
||||||
client.post(f"/api/debug-sessions/{own}/stop")
|
assert client.post(f"/api/debug-sessions/{own}/stop").status_code == 200
|
||||||
for value in (foreign, own, "not-a-session", "f" * 16, "A" * 16):
|
for value in (foreign, own, "not-a-session", "f" * 16, "A" * 16):
|
||||||
gated.records.clear()
|
gated.records.clear()
|
||||||
# Not `/api/me`: a polled route logs at DEBUG, and an un-raised request would drop the very
|
# Not `/api/me`: a polled route logs at DEBUG, and an un-raised request would drop the very
|
||||||
@@ -280,7 +283,7 @@ def test_a_header_naming_the_own_session_past_its_cap_is_ignored(gated, db, clie
|
|||||||
sid = _start(client)["id"]
|
sid = _start(client)["id"]
|
||||||
_expire(db, sid) # past its cap, not yet swept: still open in the table
|
_expire(db, sid) # past its cap, not yet swept: still open in the table
|
||||||
gated.records.clear()
|
gated.records.clear()
|
||||||
client.get("/api/debug-sessions", headers={"x-debug-session": sid})
|
assert client.get("/api/debug-sessions", headers={"x-debug-session": sid}).status_code == 200
|
||||||
(line,) = _request_lines(gated)
|
(line,) = _request_lines(gated)
|
||||||
assert getattr(line, "debug_session_id", None) is None
|
assert getattr(line, "debug_session_id", None) is None
|
||||||
assert getattr(line, "elevated", False) is False
|
assert getattr(line, "elevated", False) is False
|
||||||
@@ -616,7 +619,7 @@ def test_a_users_screenshots_are_capped_across_their_sessions(recording, monkeyp
|
|||||||
client, sid = recording
|
client, sid = recording
|
||||||
monkeypatch.setattr(debugsession, "SCREENSHOT_USER_MAX_BYTES", len(_PNG) * 2)
|
monkeypatch.setattr(debugsession, "SCREENSHOT_USER_MAX_BYTES", len(_PNG) * 2)
|
||||||
assert _upload(client, sid, _PNG).status_code == 200
|
assert _upload(client, sid, _PNG).status_code == 200
|
||||||
client.post(f"/api/debug-sessions/{sid}/stop")
|
assert client.post(f"/api/debug-sessions/{sid}/stop").status_code == 200
|
||||||
other = _start(client)["id"]
|
other = _start(client)["id"]
|
||||||
assert _upload(client, other, _PNG).status_code == 200
|
assert _upload(client, other, _PNG).status_code == 200
|
||||||
r = _upload(client, other, _PNG)
|
r = _upload(client, other, _PNG)
|
||||||
@@ -631,7 +634,7 @@ def test_new_sessions_are_paced_but_joining_one_is_not(db, make_user, as_user, m
|
|||||||
client = as_user(make_user())
|
client = as_user(make_user())
|
||||||
sid = _start(client)["id"]
|
sid = _start(client)["id"]
|
||||||
assert _start(client)["id"] == sid # another tab joining
|
assert _start(client)["id"] == sid # another tab joining
|
||||||
client.post(f"/api/debug-sessions/{sid}/stop")
|
assert client.post(f"/api/debug-sessions/{sid}/stop").status_code == 200
|
||||||
r = client.post("/api/debug-sessions", json={"level": "DEBUG"})
|
r = client.post("/api/debug-sessions", json={"level": "DEBUG"})
|
||||||
assert r.status_code == 429 and r.json()["detail"]["reason"] == "debug_session_rate_limited"
|
assert r.status_code == 429 and r.json()["detail"]["reason"] == "debug_session_rate_limited"
|
||||||
|
|
||||||
@@ -745,9 +748,16 @@ def test_the_export_is_the_saved_draft_the_evidence_and_the_screenshots(db, make
|
|||||||
db.commit()
|
db.commit()
|
||||||
_upload(client, sid, _PNG)
|
_upload(client, sid, _PNG)
|
||||||
_upload(client, sid, _JPEG)
|
_upload(client, sid, _JPEG)
|
||||||
client.put(
|
assert (
|
||||||
f"/api/debug-sessions/{sid}/draft",
|
client.put(
|
||||||
json={"title": "Spinner stays", "body": "It spins.", "excluded": [dropped_web.id, 999999]},
|
f"/api/debug-sessions/{sid}/draft",
|
||||||
|
json={
|
||||||
|
"title": "Spinner stays",
|
||||||
|
"body": "It spins.",
|
||||||
|
"excluded": [dropped_web.id, 999999],
|
||||||
|
},
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
|
|
||||||
r = client.get(f"/api/debug-sessions/{sid}/export")
|
r = client.get(f"/api/debug-sessions/{sid}/export")
|
||||||
@@ -1144,7 +1154,7 @@ def test_new_issues_are_paced_per_user(gitea_fake, db, make_user, as_user, monke
|
|||||||
first = _start(client)["id"]
|
first = _start(client)["id"]
|
||||||
_ready(client, first)
|
_ready(client, first)
|
||||||
assert client.post(f"/api/debug-sessions/{first}/issue").status_code == 200
|
assert client.post(f"/api/debug-sessions/{first}/issue").status_code == 200
|
||||||
client.post(f"/api/debug-sessions/{first}/stop")
|
assert client.post(f"/api/debug-sessions/{first}/stop").status_code == 200
|
||||||
second = _start(client)["id"]
|
second = _start(client)["id"]
|
||||||
_ready(client, second)
|
_ready(client, second)
|
||||||
r = client.post(f"/api/debug-sessions/{second}/issue")
|
r = client.post(f"/api/debug-sessions/{second}/issue")
|
||||||
|
|||||||
@@ -212,7 +212,12 @@ def test_a_timestamp_outside_the_consent_window_is_clamped(elevated):
|
|||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
future = (now + timedelta(days=365)).isoformat()
|
future = (now + timedelta(days=365)).isoformat()
|
||||||
past = (now - timedelta(days=30)).isoformat()
|
past = (now - timedelta(days=30)).isoformat()
|
||||||
client.post("/api/diagnostics/events", json={"events": [_event(ts=future), _event(ts=past)]})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/api/diagnostics/events", json={"events": [_event(ts=future), _event(ts=past)]}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
late, early = rows()
|
late, early = rows()
|
||||||
assert late.ts <= datetime.now(timezone.utc)
|
assert late.ts <= datetime.now(timezone.utc)
|
||||||
assert early.ts >= now - timedelta(hours=diagnostics.ELEVATION_HOURS, minutes=1)
|
assert early.ts >= now - timedelta(hours=diagnostics.ELEVATION_HOURS, minutes=1)
|
||||||
@@ -342,15 +347,15 @@ def _app() -> TestClient:
|
|||||||
|
|
||||||
|
|
||||||
def test_the_account_reaches_the_handlers_own_lines(gated):
|
def test_the_account_reaches_the_handlers_own_lines(gated):
|
||||||
_app().get("/api/thing/41")
|
assert _app().get("/api/thing/41").status_code == 200
|
||||||
line = next(r for r in gated.records if r.getMessage() == "an ordinary line")
|
line = next(r for r in gated.records if r.getMessage() == "an ordinary line")
|
||||||
assert getattr(line, "user_id", None) == 41
|
assert getattr(line, "user_id", None) == 41
|
||||||
|
|
||||||
|
|
||||||
def test_one_users_elevation_logs_their_requests_in_detail_and_nobody_elses(gated):
|
def test_one_users_elevation_logs_their_requests_in_detail_and_nobody_elses(gated):
|
||||||
client = _app()
|
client = _app()
|
||||||
client.get("/api/thing/7", params={"level": "DEBUG"})
|
assert client.get("/api/thing/7", params={"level": "DEBUG"}).status_code == 200
|
||||||
client.get("/api/thing/8")
|
assert client.get("/api/thing/8").status_code == 200
|
||||||
detail = [r for r in gated.records if r.getMessage().startswith("detail only")]
|
detail = [r for r in gated.records if r.getMessage().startswith("detail only")]
|
||||||
assert [r.user_id for r in detail] == [7]
|
assert [r.user_id for r in detail] == [7]
|
||||||
assert getattr(detail[0], "elevated", False) is True
|
assert getattr(detail[0], "elevated", False) is True
|
||||||
@@ -363,9 +368,9 @@ def test_one_users_elevation_logs_their_requests_in_detail_and_nobody_elses(gate
|
|||||||
|
|
||||||
def test_the_root_level_returns_to_the_base_when_the_elevation_ends(gated):
|
def test_the_root_level_returns_to_the_base_when_the_elevation_ends(gated):
|
||||||
client = _app()
|
client = _app()
|
||||||
client.get("/api/thing/7", params={"level": "TRACE"})
|
assert client.get("/api/thing/7", params={"level": "TRACE"}).status_code == 200
|
||||||
assert logging.getLogger().level == logs.TRACE
|
assert logging.getLogger().level == logs.TRACE
|
||||||
client.get("/api/thing/7") # the same user, back at the base
|
assert client.get("/api/thing/7").status_code == 200 # the same user, back at the base
|
||||||
assert logging.getLogger().level == logging.INFO
|
assert logging.getLogger().level == logging.INFO
|
||||||
logs._track_elevation(9, logging.DEBUG, datetime.now(timezone.utc) - timedelta(seconds=1))
|
logs._track_elevation(9, logging.DEBUG, datetime.now(timezone.utc) - timedelta(seconds=1))
|
||||||
assert logging.getLogger().level == logging.INFO # an already-expired one is pruned at once
|
assert logging.getLogger().level == logging.INFO # an already-expired one is pruned at once
|
||||||
@@ -373,15 +378,25 @@ def test_the_root_level_returns_to_the_base_when_the_elevation_ends(gated):
|
|||||||
|
|
||||||
def test_the_query_string_is_logged_only_for_a_traced_request_and_redacted(gated):
|
def test_the_query_string_is_logged_only_for_a_traced_request_and_redacted(gated):
|
||||||
client = _app()
|
client = _app()
|
||||||
client.get("/api/thing/7", params={"level": "DEBUG", "q": "lofi"})
|
assert client.get("/api/thing/7", params={"level": "DEBUG", "q": "lofi"}).status_code == 200
|
||||||
client.get("/api/thing/7", params={"level": "TRACE", "q": "lofi", "token": "SECRET"})
|
assert (
|
||||||
|
client.get(
|
||||||
|
"/api/thing/7", params={"level": "TRACE", "q": "lofi", "token": "SECRET"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
lines = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
lines = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||||
assert "query" not in lines[0].attrs
|
assert "query" not in lines[0].attrs
|
||||||
assert "q=lofi" in lines[1].attrs["query"] and "SECRET" not in lines[1].attrs["query"]
|
assert "q=lofi" in lines[1].attrs["query"] and "SECRET" not in lines[1].attrs["query"]
|
||||||
|
|
||||||
|
|
||||||
def test_an_address_typed_into_a_search_is_redacted_though_the_browser_encodes_it(gated):
|
def test_an_address_typed_into_a_search_is_redacted_though_the_browser_encodes_it(gated):
|
||||||
_app().get("/api/thing/7?level=TRACE&q=someone%40example.com&new_password=hunter2")
|
assert (
|
||||||
|
_app()
|
||||||
|
.get("/api/thing/7?level=TRACE&q=someone%40example.com&new_password=hunter2")
|
||||||
|
.status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||||
assert line.attrs["query"] == "level=TRACE&q=[email]&new_password=[redacted]"
|
assert line.attrs["query"] == "level=TRACE&q=[email]&new_password=[redacted]"
|
||||||
|
|
||||||
@@ -400,7 +415,7 @@ def test_a_credential_param_goes_whole_however_its_value_decodes(raw):
|
|||||||
|
|
||||||
def test_an_admin_level_of_trace_does_not_log_other_users_query_strings(gated):
|
def test_an_admin_level_of_trace_does_not_log_other_users_query_strings(gated):
|
||||||
logs.set_level("TRACE")
|
logs.set_level("TRACE")
|
||||||
_app().get("/api/thing/7", params={"q": "something private"})
|
assert _app().get("/api/thing/7", params={"q": "something private"}).status_code == 200
|
||||||
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
(line,) = [r for r in gated.records if getattr(r, "event", None) == "http.request"]
|
||||||
assert "query" not in line.attrs
|
assert "query" not in line.attrs
|
||||||
|
|
||||||
@@ -488,7 +503,7 @@ def test_a_real_sign_in_carries_the_consented_level_into_the_process(gated, clie
|
|||||||
user = _signed_in(client, make_user)
|
user = _signed_in(client, make_user)
|
||||||
assert client.put("/api/me/diagnostics", json={"level": "DEBUG"}).status_code == 200
|
assert client.put("/api/me/diagnostics", json={"level": "DEBUG"}).status_code == 200
|
||||||
logs._elevations.clear() # only a request resolving the account through auth may put it back
|
logs._elevations.clear() # only a request resolving the account through auth may put it back
|
||||||
client.get("/api/me")
|
assert client.get("/api/me").status_code == 200
|
||||||
assert logs._elevations[user.id][0] == logging.DEBUG
|
assert logs._elevations[user.id][0] == logging.DEBUG
|
||||||
assert client.put("/api/me/diagnostics", json={"level": "INFO"}).status_code == 200
|
assert client.put("/api/me/diagnostics", json={"level": "INFO"}).status_code == 200
|
||||||
assert user.id not in logs._elevations
|
assert user.id not in logs._elevations
|
||||||
|
|||||||
@@ -13,12 +13,12 @@ hold for Google Workspace custom domains unless their admin enabled it.
|
|||||||
import pytest
|
import pytest
|
||||||
from sqlalchemy import select, text
|
from sqlalchemy import select, text
|
||||||
|
|
||||||
_UAT_PW = "correct horse battery"
|
|
||||||
|
|
||||||
from app.auth import is_allowed
|
from app.auth import is_allowed
|
||||||
from app.models import Invite, User
|
from app.models import Invite, User
|
||||||
from app.utils import canonical_email
|
from app.utils import canonical_email
|
||||||
|
|
||||||
|
_UAT_PW = "correct horse battery"
|
||||||
|
|
||||||
# (input, expected canonical) — the matrix the sprint promised, plus the shapes that must NOT fold.
|
# (input, expected canonical) — the matrix the sprint promised, plus the shapes that must NOT fold.
|
||||||
CASES = [
|
CASES = [
|
||||||
# Gmail: dots and +tags are noise…
|
# Gmail: dots and +tags are noise…
|
||||||
@@ -334,7 +334,7 @@ def test_me_reports_the_google_address_only_when_it_differs(client, db, make_use
|
|||||||
)
|
)
|
||||||
assert client.get("/api/me").json()["google_email"] is None
|
assert client.get("/api/me").json()["google_email"] is None
|
||||||
|
|
||||||
client.post("/auth/logout")
|
assert client.post("/auth/logout").status_code == 200
|
||||||
tagged = make_user(email="me+siftlode@gmail.com", password=_UAT_PW)
|
tagged = make_user(email="me+siftlode@gmail.com", password=_UAT_PW)
|
||||||
tagged.google_email = "me@gmail.com"
|
tagged.google_email = "me@gmail.com"
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|||||||
@@ -0,0 +1,259 @@
|
|||||||
|
"""An empty answer is not an emptied library (E32 S2).
|
||||||
|
|
||||||
|
Every mirror here reconciles: what the remote no longer lists is removed locally. The catalog
|
||||||
|
(`_prune_stale`) and the subscriptions (`should_prune`) had long refused to do that on an empty
|
||||||
|
listing that contradicts what we hold — a 200 with no items is a blip far more often than a user
|
||||||
|
who removed everything. Two mirrors did not: the Plex collections (a row carries the user-set
|
||||||
|
`editable` flag, and every member its keys) and the YouTube playlist mirrors. These hold both — and,
|
||||||
|
one level down (E32 review), a collection's members and a playlist's items: an empty answer that
|
||||||
|
contradicts the remote's own count is not believed either.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.models import (
|
||||||
|
Channel,
|
||||||
|
Playlist,
|
||||||
|
PlaylistItem,
|
||||||
|
PlexCollection,
|
||||||
|
PlexItem,
|
||||||
|
PlexLibrary,
|
||||||
|
Video,
|
||||||
|
)
|
||||||
|
from app.plex import sync as plex_sync
|
||||||
|
from app.sync import playlists as plsync
|
||||||
|
from app.youtube.client import YouTubeError
|
||||||
|
|
||||||
|
|
||||||
|
class _Plex:
|
||||||
|
def __init__(self, collections: list[dict], children: dict[str, list[dict]]):
|
||||||
|
self._collections = collections
|
||||||
|
self._children = children
|
||||||
|
|
||||||
|
def collections(self, section_key):
|
||||||
|
return self._collections
|
||||||
|
|
||||||
|
def collection_children(self, rk):
|
||||||
|
return self._children.get(rk, [])
|
||||||
|
|
||||||
|
def metadata(self, rk):
|
||||||
|
return next((c for c in self._collections if c.get("ratingKey") == rk), {})
|
||||||
|
|
||||||
|
|
||||||
|
def _library_with_a_collection(db):
|
||||||
|
lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||||
|
db.add(lib)
|
||||||
|
db.flush()
|
||||||
|
db.add(PlexCollection(rating_key="c1", library_id=lib.id, title="Bond", editable=True))
|
||||||
|
movie = PlexItem(
|
||||||
|
rating_key="m1", library_id=lib.id, kind="movie", title="Dr. No", collection_keys=["c1"]
|
||||||
|
)
|
||||||
|
db.add(movie)
|
||||||
|
db.commit()
|
||||||
|
return lib, movie
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_empty_collection_listing_keeps_the_collections_and_their_members(db):
|
||||||
|
lib, movie = _library_with_a_collection(db)
|
||||||
|
stats = {"collections": 0}
|
||||||
|
plex_sync._sync_collections(db, _Plex([], {}), lib, stats)
|
||||||
|
db.commit()
|
||||||
|
db.expire_all()
|
||||||
|
col = db.execute(select(PlexCollection).where(PlexCollection.rating_key == "c1")).scalar_one()
|
||||||
|
assert col.editable is True
|
||||||
|
assert db.get(PlexItem, movie.id).collection_keys == ["c1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_listing_with_something_in_it_still_prunes_what_it_no_longer_has(db):
|
||||||
|
lib, movie = _library_with_a_collection(db)
|
||||||
|
stats = {"collections": 0}
|
||||||
|
other = {"ratingKey": "c2", "title": "Alien"}
|
||||||
|
plex_sync._sync_collections(db, _Plex([other], {"c2": []}), lib, stats)
|
||||||
|
db.commit()
|
||||||
|
db.expire_all()
|
||||||
|
keys = set(db.execute(select(PlexCollection.rating_key)).scalars())
|
||||||
|
assert keys == {"c2"}
|
||||||
|
assert db.get(PlexItem, movie.id).collection_keys is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_members_from_a_collection_plex_counts_as_non_empty_keeps_its_members(db):
|
||||||
|
lib, movie = _library_with_a_collection(db)
|
||||||
|
stats = {"collections": 0}
|
||||||
|
bond = {"ratingKey": "c1", "title": "Bond", "childCount": 2}
|
||||||
|
plex_sync._sync_collections(db, _Plex([bond], {"c1": []}), lib, stats)
|
||||||
|
db.commit()
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(PlexItem, movie.id).collection_keys == ["c1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_collection_plex_counts_as_empty_does_lose_its_members(db):
|
||||||
|
lib, movie = _library_with_a_collection(db)
|
||||||
|
stats = {"collections": 0}
|
||||||
|
bond = {"ratingKey": "c1", "title": "Bond", "childCount": 0}
|
||||||
|
plex_sync._sync_collections(db, _Plex([bond], {"c1": []}), lib, stats)
|
||||||
|
db.commit()
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(PlexItem, movie.id).collection_keys is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(("count", "kept"), [(2, ["c1"]), (0, None)])
|
||||||
|
def test_a_collection_resync_believes_an_empty_member_list_only_with_a_zero_count(db, count, kept):
|
||||||
|
lib, movie = _library_with_a_collection(db)
|
||||||
|
bond = {"ratingKey": "c1", "title": "Bond", "childCount": count}
|
||||||
|
plex_sync.resync_collection(db, _Plex([bond], {"c1": []}), lib, "c1")
|
||||||
|
db.commit()
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(PlexItem, movie.id).collection_keys == kept
|
||||||
|
|
||||||
|
|
||||||
|
class _YT:
|
||||||
|
def __init__(self, playlists: list[dict], snippet: dict | None = None):
|
||||||
|
self._playlists = playlists
|
||||||
|
self._snippet = snippet
|
||||||
|
|
||||||
|
def __call__(self, *_):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def iter_my_playlists(self):
|
||||||
|
return self._playlists
|
||||||
|
|
||||||
|
def iter_playlist_items_with_ids(self, _ytid):
|
||||||
|
return []
|
||||||
|
|
||||||
|
def iter_my_playlist_video_ids(self, _ytid):
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_playlist_snippet(self, _ytid):
|
||||||
|
return self._snippet
|
||||||
|
|
||||||
|
def get_videos(self, _ids):
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_empty_playlist_answer_keeps_the_youtube_mirrors(db, make_user, monkeypatch):
|
||||||
|
user = make_user()
|
||||||
|
db.add(Playlist(user_id=user.id, name="Mixes", source="youtube", yt_playlist_id="PL1"))
|
||||||
|
db.commit()
|
||||||
|
monkeypatch.setattr(plsync, "YouTubeClient", _YT([]))
|
||||||
|
plsync.pull_user_playlists(db, user)
|
||||||
|
db.expire_all()
|
||||||
|
names = db.execute(select(Playlist.name).where(Playlist.user_id == user.id)).scalars().all()
|
||||||
|
assert "Mixes" in names
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_playlist_answer_with_something_in_it_still_drops_a_vanished_mirror(
|
||||||
|
db, make_user, monkeypatch
|
||||||
|
):
|
||||||
|
user = make_user()
|
||||||
|
db.add(Playlist(user_id=user.id, name="Mixes", source="youtube", yt_playlist_id="PL1"))
|
||||||
|
db.commit()
|
||||||
|
monkeypatch.setattr(plsync, "YouTubeClient", _YT([{"id": "PL2", "title": "New"}]))
|
||||||
|
plsync.pull_user_playlists(db, user)
|
||||||
|
db.expire_all()
|
||||||
|
ytids = set(
|
||||||
|
db.execute(select(Playlist.yt_playlist_id).where(Playlist.source == "youtube")).scalars()
|
||||||
|
)
|
||||||
|
assert "PL1" not in ytids
|
||||||
|
|
||||||
|
|
||||||
|
class _YTOneGone(_YT):
|
||||||
|
"""A list deleted on YouTube between the listing and the read of its items."""
|
||||||
|
|
||||||
|
def iter_my_playlist_video_ids(self, ytid):
|
||||||
|
if ytid == "PLgone":
|
||||||
|
raise YouTubeError("m", reason="playlistNotFound", path="playlistItems")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_list_deleted_mid_pull_is_skipped_and_the_rest_still_sync(db, make_user, monkeypatch):
|
||||||
|
# Not "this playlist no longer exists" for a bulk pull the user never named a list in, and not
|
||||||
|
# a failure for every other list either (E32 review round 2): the next pull drops the mirror.
|
||||||
|
user = make_user()
|
||||||
|
db.commit()
|
||||||
|
listing = [
|
||||||
|
{"id": "PLgone", "title": "Gone", "item_count": 3},
|
||||||
|
{"id": "PL2", "title": "Kept", "item_count": 0},
|
||||||
|
]
|
||||||
|
monkeypatch.setattr(plsync, "YouTubeClient", _YTOneGone(listing))
|
||||||
|
assert plsync.pull_user_playlists(db, user) == {"synced": 1}
|
||||||
|
db.expire_all()
|
||||||
|
ytids = set(
|
||||||
|
db.execute(select(Playlist.yt_playlist_id).where(Playlist.user_id == user.id)).scalars()
|
||||||
|
)
|
||||||
|
assert ytids == {"PL2"}
|
||||||
|
|
||||||
|
|
||||||
|
def _mirror_with_an_item(db, user, **kw):
|
||||||
|
db.add(Channel(id="UC_pl", title="C"))
|
||||||
|
db.flush()
|
||||||
|
db.add(Video(id="v1", channel_id="UC_pl", title="One"))
|
||||||
|
pl = Playlist(user_id=user.id, name="Mixes", yt_playlist_id="PL1", **kw)
|
||||||
|
db.add(pl)
|
||||||
|
db.flush()
|
||||||
|
db.add(PlaylistItem(playlist_id=pl.id, video_id="v1", position=0))
|
||||||
|
db.commit()
|
||||||
|
return pl
|
||||||
|
|
||||||
|
|
||||||
|
def _items(db, pl):
|
||||||
|
db.expire_all()
|
||||||
|
return (
|
||||||
|
db.execute(select(PlaylistItem.video_id).where(PlaylistItem.playlist_id == pl.id))
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_items_from_a_playlist_youtube_counts_as_non_empty_keeps_the_mirror_items(
|
||||||
|
db, make_user, monkeypatch
|
||||||
|
):
|
||||||
|
user = make_user()
|
||||||
|
pl = _mirror_with_an_item(db, user, source="youtube")
|
||||||
|
listing = [{"id": "PL1", "title": "Mixes", "item_count": 1}]
|
||||||
|
monkeypatch.setattr(plsync, "YouTubeClient", _YT(listing))
|
||||||
|
plsync.pull_user_playlists(db, user)
|
||||||
|
assert _items(db, pl) == ["v1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_playlist_youtube_counts_as_empty_is_emptied(db, make_user, monkeypatch):
|
||||||
|
user = make_user()
|
||||||
|
pl = _mirror_with_an_item(db, user, source="youtube")
|
||||||
|
listing = [{"id": "PL1", "title": "Mixes", "item_count": 0}]
|
||||||
|
monkeypatch.setattr(plsync, "YouTubeClient", _YT(listing))
|
||||||
|
plsync.pull_user_playlists(db, user)
|
||||||
|
assert _items(db, pl) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_reset_to_youtube_refuses_a_blank_listing_and_keeps_the_items(db, make_user, monkeypatch):
|
||||||
|
user = make_user()
|
||||||
|
pl = _mirror_with_an_item(db, user, source="youtube")
|
||||||
|
monkeypatch.setattr(plsync, "YouTubeClient", _YT([], {"title": "Mixes", "item_count": 1}))
|
||||||
|
with pytest.raises(YouTubeError) as err:
|
||||||
|
plsync.repull_playlist(db, user, pl)
|
||||||
|
db.rollback()
|
||||||
|
assert err.value.reason == "emptyAnswer"
|
||||||
|
assert _items(db, pl) == ["v1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_reset_of_a_deleted_playlist_says_it_is_gone(db, make_user, monkeypatch):
|
||||||
|
user = make_user()
|
||||||
|
pl = _mirror_with_an_item(db, user, source="youtube")
|
||||||
|
monkeypatch.setattr(plsync, "YouTubeClient", _YT([], None))
|
||||||
|
with pytest.raises(YouTubeError) as err:
|
||||||
|
plsync.repull_playlist(db, user, pl)
|
||||||
|
assert err.value.reason == "playlistNotFound"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_push_plan_refuses_a_blank_listing_rather_than_reading_it_as_all_deleted(db, make_user):
|
||||||
|
user = make_user()
|
||||||
|
pl = _mirror_with_an_item(db, user, source="local")
|
||||||
|
with pytest.raises(YouTubeError) as err:
|
||||||
|
plsync.plan_merge(db, user, pl, live=([], {"title": "Mixes", "item_count": 1}))
|
||||||
|
assert err.value.reason == "emptyAnswer"
|
||||||
@@ -12,6 +12,8 @@ from datetime import datetime, timezone
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from app import feedquery
|
||||||
|
|
||||||
from app.models import Channel, ChannelTag, Subscription, Tag, Video, VideoState
|
from app.models import Channel, ChannelTag, Subscription, Tag, Video, VideoState
|
||||||
|
|
||||||
|
|
||||||
@@ -168,9 +170,50 @@ _SUMMARY_CASES = [
|
|||||||
"show=all&untagged=true&tags={L1}",
|
"show=all&untagged=true&tags={L1}",
|
||||||
"show=all&q=clip",
|
"show=all&q=clip",
|
||||||
"show=unwatched",
|
"show=unwatched",
|
||||||
|
# A deleted tag's id (a saved view or the feed state still holds it) matches nothing.
|
||||||
|
"show=all&tags=987654",
|
||||||
|
"show=all&tags={L1}&tags=987654",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_deleted_tags_id_filters_to_nothing_not_to_everything(db, as_user, make_user):
|
||||||
|
"""A saved view or the feed's filter state keeps a tag id after the tag is deleted. The feed used
|
||||||
|
to drop an unknown id, so that view silently showed the WHOLE feed under its old name (E32 S2).
|
||||||
|
A filter on a tag that no longer exists matches nothing; the chip says the tag is gone."""
|
||||||
|
user, tag_ids = _seed_tagged(db, make_user)
|
||||||
|
client = as_user(user)
|
||||||
|
everything = client.get("/api/feed/count?show=all")
|
||||||
|
assert everything.status_code == 200
|
||||||
|
assert everything.json()["count"] == 8
|
||||||
|
for qs in ("show=all&tags=987654", f"show=all&tags={tag_ids['L1']}&tags=987654"):
|
||||||
|
r = client.get(f"/api/feed/count?{qs}")
|
||||||
|
assert r.status_code == 200
|
||||||
|
assert r.json()["count"] == 0, qs
|
||||||
|
r = client.get(f"/api/feed?{qs}")
|
||||||
|
assert r.status_code == 200
|
||||||
|
assert r.json()["items"] == [], qs
|
||||||
|
|
||||||
|
|
||||||
|
def test_another_users_private_tag_answers_like_one_that_does_not_exist(db, as_user, make_user):
|
||||||
|
"""The category split resolved any tag id, so a foreign PRIVATE tag landed in its real category
|
||||||
|
and the summary's facets differed from a non-existent id's: an oracle for which foreign ids exist
|
||||||
|
(E32 review). Both must now read as "missing"."""
|
||||||
|
user, tag_ids = _seed_tagged(db, make_user)
|
||||||
|
stranger = make_user(email="stranger@example.com")
|
||||||
|
private = Tag(user_id=stranger.id, name="Secret", category="language")
|
||||||
|
db.add(private)
|
||||||
|
db.commit()
|
||||||
|
assert feedquery.tags_by_category(db, user, [tag_ids["L1"], private.id]) == {
|
||||||
|
"language": [tag_ids["L1"]],
|
||||||
|
feedquery.MISSING_TAG_CATEGORY: [private.id],
|
||||||
|
}
|
||||||
|
client = as_user(user)
|
||||||
|
foreign = client.get(f"/api/feed/summary?show=all&tags={tag_ids['T1']}&tags={private.id}")
|
||||||
|
absent = client.get(f"/api/feed/summary?show=all&tags={tag_ids['T1']}&tags=987654")
|
||||||
|
assert foreign.status_code == absent.status_code == 200
|
||||||
|
assert foreign.json() == absent.json()
|
||||||
|
|
||||||
|
|
||||||
def test_untagged_filter_keeps_only_channels_the_user_has_not_tagged(db, as_user, make_user):
|
def test_untagged_filter_keeps_only_channels_the_user_has_not_tagged(db, as_user, make_user):
|
||||||
"""The "No tag" filter (item B): `untagged=true` keeps only channels with NO tag the USER personally
|
"""The "No tag" filter (item B): `untagged=true` keeps only channels with NO tag the USER personally
|
||||||
attached (no user-owned ChannelTag). System/auto tags (user_id NULL) don't count, and another user's
|
attached (no user-owned ChannelTag). System/auto tags (user_id NULL) don't count, and another user's
|
||||||
|
|||||||
@@ -100,7 +100,12 @@ def test_the_admin_user_list_carries_last_login(as_user, db, make_user):
|
|||||||
def test_the_account_sees_its_own_last_sign_in(client, db, make_user):
|
def test_the_account_sees_its_own_last_sign_in(client, db, make_user):
|
||||||
# Settings → Account shows it (prod UAT 2026-09-19: only the admin's user list did).
|
# Settings → Account shows it (prod UAT 2026-09-19: only the admin's user list did).
|
||||||
make_user(email="me@example.com", password=_PW)
|
make_user(email="me@example.com", password=_PW)
|
||||||
client.post("/auth/password-login", json={"email": "me@example.com", "password": _PW})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/auth/password-login", json={"email": "me@example.com", "password": _PW}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
|
||||||
stamp = client.get("/api/me").json()["last_login_at"]
|
stamp = client.get("/api/me").json()["last_login_at"]
|
||||||
|
|
||||||
|
|||||||
@@ -277,7 +277,7 @@ def test_a_signed_in_request_is_never_capped_and_names_its_account(sink):
|
|||||||
|
|
||||||
client = TestClient(app)
|
client = TestClient(app)
|
||||||
for _ in range(logs.ANONYMOUS_LINES_PER_MINUTE + 3):
|
for _ in range(logs.ANONYMOUS_LINES_PER_MINUTE + 3):
|
||||||
client.get("/api/me")
|
assert client.get("/api/me").status_code == 200
|
||||||
records = sink.records[-(logs.ANONYMOUS_LINES_PER_MINUTE + 3) :]
|
records = sink.records[-(logs.ANONYMOUS_LINES_PER_MINUTE + 3) :]
|
||||||
assert all(getattr(r, logs.PERSIST, True) for r in records)
|
assert all(getattr(r, logs.PERSIST, True) for r in records)
|
||||||
assert sink.objects()[-1]["user_id"] == 42 and "user_id" not in sink.objects()[-1]["attrs"]
|
assert sink.objects()[-1]["user_id"] == 42 and "user_id" not in sink.objects()[-1]["attrs"]
|
||||||
@@ -325,9 +325,9 @@ def test_the_log_consoles_own_reads_are_debug_but_its_failures_are_not(sink):
|
|||||||
http = logging.getLogger("siftlode.http")
|
http = logging.getLogger("siftlode.http")
|
||||||
http.setLevel(logs.TRACE)
|
http.setLevel(logs.TRACE)
|
||||||
client = _client()
|
client = _client()
|
||||||
client.get("/api/admin/logs/facets")
|
assert client.get("/api/admin/logs/facets").status_code == 200
|
||||||
assert sink.objects()[-1]["level"] == "DEBUG"
|
assert sink.objects()[-1]["level"] == "DEBUG"
|
||||||
client.get("/api/admin/logs/facets?fail=1")
|
assert client.get("/api/admin/logs/facets?fail=1").status_code == 400
|
||||||
assert sink.objects()[-1]["level"] == "WARNING"
|
assert sink.objects()[-1]["level"] == "WARNING"
|
||||||
|
|
||||||
|
|
||||||
@@ -344,7 +344,7 @@ def test_a_polled_read_is_debug_a_write_to_the_same_route_is_not(sink):
|
|||||||
r = client.post("/api/messages/unread_count")
|
r = client.post("/api/messages/unread_count")
|
||||||
assert sink.objects()[-1]["level"] == "INFO"
|
assert sink.objects()[-1]["level"] == "INFO"
|
||||||
assert "X-Log-Quiet" not in r.headers
|
assert "X-Log-Quiet" not in r.headers
|
||||||
client.get("/api/videos/abc")
|
assert client.get("/api/videos/abc").status_code == 200
|
||||||
assert sink.objects()[-1]["level"] == "INFO"
|
assert sink.objects()[-1]["level"] == "INFO"
|
||||||
|
|
||||||
|
|
||||||
@@ -360,7 +360,7 @@ def test_every_polled_route_is_a_route_the_app_has():
|
|||||||
|
|
||||||
def test_a_failing_request_is_logged_by_status_and_an_unhandled_error_is_logged_at_all(sink):
|
def test_a_failing_request_is_logged_by_status_and_an_unhandled_error_is_logged_at_all(sink):
|
||||||
client = _client()
|
client = _client()
|
||||||
client.get("/api/videos/missing/deeper") # 404 from the router
|
assert client.get("/api/videos/missing/deeper").status_code == 404 # 404 from the router
|
||||||
assert sink.objects()[-1]["level"] == "WARNING"
|
assert sink.objects()[-1]["level"] == "WARNING"
|
||||||
client.get("/boom")
|
client.get("/boom")
|
||||||
payload = sink.objects()[-1]
|
payload = sink.objects()[-1]
|
||||||
|
|||||||
@@ -259,7 +259,7 @@ def test_the_writer_drains_what_was_queued_when_it_is_stopped(db):
|
|||||||
log.addHandler(sink)
|
log.addHandler(sink)
|
||||||
try:
|
try:
|
||||||
for i in range(logsink.BATCH_MAX + 5):
|
for i in range(logsink.BATCH_MAX + 5):
|
||||||
# noqa G004: the f-string is the POINT — a distinct rendered message per line is
|
# G004: the f-string is the POINT — a distinct rendered message per line is
|
||||||
# what keeps the flood limiter (which groups by template) out of this case.
|
# what keeps the flood limiter (which groups by template) out of this case.
|
||||||
log.warning(f"line {i}") # noqa: G004
|
log.warning(f"line {i}") # noqa: G004
|
||||||
finally:
|
finally:
|
||||||
|
|||||||
@@ -154,8 +154,18 @@ def test_an_access_request_records_the_language_it_was_made_in(db, client):
|
|||||||
|
|
||||||
|
|
||||||
def test_asking_again_from_another_language_updates_the_row(db, client):
|
def test_asking_again_from_another_language_updates_the_row(db, client):
|
||||||
client.post("/auth/request-access", json={"email": "asking@example.com", "lang": "en"})
|
assert (
|
||||||
client.post("/auth/request-access", json={"email": "asking@example.com", "lang": "hu"})
|
client.post(
|
||||||
|
"/auth/request-access", json={"email": "asking@example.com", "lang": "en"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/auth/request-access", json={"email": "asking@example.com", "lang": "hu"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one()
|
inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one()
|
||||||
@@ -165,7 +175,12 @@ def test_asking_again_from_another_language_updates_the_row(db, client):
|
|||||||
|
|
||||||
|
|
||||||
def test_an_unsupported_language_is_ignored_rather_than_stored(db, client):
|
def test_an_unsupported_language_is_ignored_rather_than_stored(db, client):
|
||||||
client.post("/auth/request-access", json={"email": "asking@example.com", "lang": "../../etc"})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/auth/request-access", json={"email": "asking@example.com", "lang": "../../etc"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
|
||||||
inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one()
|
inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one()
|
||||||
assert inv.lang is None
|
assert inv.lang is None
|
||||||
@@ -194,7 +209,12 @@ def test_the_confirmation_mail_is_written_in_the_accounts_language(db, client, m
|
|||||||
monkeypatch.setattr(email_mod, "email_enabled", lambda db=None: True)
|
monkeypatch.setattr(email_mod, "email_enabled", lambda db=None: True)
|
||||||
sent = _bodies(monkeypatch)
|
sent = _bodies(monkeypatch)
|
||||||
|
|
||||||
client.post("/auth/register", json={"email": "hu@example.com", "password": _PW, "lang": "hu"})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/auth/register", json={"email": "hu@example.com", "password": _PW, "lang": "hu"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
|
||||||
(_, subject, body) = sent[0]
|
(_, subject, body) = sent[0]
|
||||||
assert subject == mailtext.MAIL_TEXT["verify_email.subject"]["hu"]
|
assert subject == mailtext.MAIL_TEXT["verify_email.subject"]["hu"]
|
||||||
@@ -209,7 +229,12 @@ def test_the_approval_mail_answers_in_the_language_the_request_was_made_in(
|
|||||||
monkeypatch.setattr(email_mod, "email_enabled", lambda db=None: True)
|
monkeypatch.setattr(email_mod, "email_enabled", lambda db=None: True)
|
||||||
sent = _bodies(monkeypatch)
|
sent = _bodies(monkeypatch)
|
||||||
admin = make_user(email="admin@example.com", role="admin")
|
admin = make_user(email="admin@example.com", role="admin")
|
||||||
client.post("/auth/request-access", json={"email": "asking@example.com", "lang": "hu"})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/auth/request-access", json={"email": "asking@example.com", "lang": "hu"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one()
|
inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one()
|
||||||
sent.clear() # drop the admin notification; this test is about the reply to the requester
|
sent.clear() # drop the admin notification; this test is about the reply to the requester
|
||||||
|
|
||||||
@@ -227,11 +252,16 @@ def test_a_denial_answers_in_that_language_too(db, client, make_user, as_user, m
|
|||||||
monkeypatch.setattr(email_mod, "email_enabled", lambda db=None: True)
|
monkeypatch.setattr(email_mod, "email_enabled", lambda db=None: True)
|
||||||
sent = _bodies(monkeypatch)
|
sent = _bodies(monkeypatch)
|
||||||
admin = make_user(email="admin@example.com", role="admin")
|
admin = make_user(email="admin@example.com", role="admin")
|
||||||
client.post("/auth/request-access", json={"email": "asking@example.com", "lang": "hu"})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/auth/request-access", json={"email": "asking@example.com", "lang": "hu"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one()
|
inv = db.execute(select(Invite).where(Invite.email == "asking@example.com")).scalar_one()
|
||||||
sent.clear()
|
sent.clear()
|
||||||
|
|
||||||
as_user(admin).post(f"/api/admin/invites/{inv.id}/deny")
|
assert as_user(admin).post(f"/api/admin/invites/{inv.id}/deny").status_code == 200
|
||||||
|
|
||||||
assert sent[0][1] == mailtext.MAIL_TEXT["access_denied.subject"]["hu"]
|
assert sent[0][1] == mailtext.MAIL_TEXT["access_denied.subject"]["hu"]
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,182 @@
|
|||||||
|
"""sync/maintenance — the job that HARD-DELETES videos (E32 S2: 19% covered).
|
||||||
|
|
||||||
|
Its lifecycle: a video missing from videos.list is flagged and hidden; after the grace period a
|
||||||
|
still-missing one is deleted (cascading to every playlist that held it) and each affected user
|
||||||
|
gets one notification; a video that answers again is unhidden. A live broadcast is never a delete
|
||||||
|
trigger, and an upcoming premiere that never went live is. YouTube is a fake that answers
|
||||||
|
videos.list from a dict.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import timedelta
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app.models import Channel, Notification, Playlist, PlaylistItem, Video
|
||||||
|
from app.sync import maintenance
|
||||||
|
from app.utils import now_utc
|
||||||
|
|
||||||
|
|
||||||
|
class _YT:
|
||||||
|
def __init__(self, items: dict[str, dict]):
|
||||||
|
self.items = items
|
||||||
|
self.asked: list[list[str]] = []
|
||||||
|
|
||||||
|
def get_videos(self, ids):
|
||||||
|
self.asked.append(list(ids))
|
||||||
|
return [self.items[i] for i in ids if i in self.items]
|
||||||
|
|
||||||
|
|
||||||
|
def _item(vid: str, **snippet) -> dict:
|
||||||
|
return {"id": vid, "snippet": {"title": vid, **snippet}}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def channel(db):
|
||||||
|
ch = Channel(id="UC_maint", title="Maint")
|
||||||
|
db.add(ch)
|
||||||
|
db.flush()
|
||||||
|
return ch
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_flagged_video_that_answers_again_is_unhidden(db, channel):
|
||||||
|
v = Video(
|
||||||
|
id="v1", channel_id=channel.id, unavailable_since=now_utc(), unavailable_reason="removed"
|
||||||
|
)
|
||||||
|
db.add(v)
|
||||||
|
db.commit()
|
||||||
|
out = maintenance._recheck_flagged(db, _YT({"v1": _item("v1")}), now_utc())
|
||||||
|
db.refresh(v)
|
||||||
|
assert v.unavailable_since is None and v.unavailable_reason is None
|
||||||
|
assert out == {"checked": 1, "recovered": 1, "deleted": 0, "notified": 0}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_video_still_missing_inside_its_grace_period_is_kept_hidden(db, channel):
|
||||||
|
v = Video(
|
||||||
|
id="v1",
|
||||||
|
channel_id=channel.id,
|
||||||
|
unavailable_since=now_utc() - timedelta(days=maintenance.DEFAULT_GRACE_DAYS - 1),
|
||||||
|
unavailable_reason="removed",
|
||||||
|
)
|
||||||
|
db.add(v)
|
||||||
|
db.commit()
|
||||||
|
out = maintenance._recheck_flagged(db, _YT({}), now_utc())
|
||||||
|
assert db.get(Video, "v1") is not None
|
||||||
|
assert out["deleted"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_past_the_grace_period_it_is_deleted_and_each_user_who_kept_it_told_once(
|
||||||
|
db, channel, make_user
|
||||||
|
):
|
||||||
|
a, b = make_user(), make_user()
|
||||||
|
gone_since = now_utc() - timedelta(days=maintenance.DEFAULT_GRACE_DAYS + 1)
|
||||||
|
for vid in ("v1", "v2"):
|
||||||
|
db.add(
|
||||||
|
Video(
|
||||||
|
id=vid,
|
||||||
|
channel_id=channel.id,
|
||||||
|
title=f"T {vid}",
|
||||||
|
unavailable_since=gone_since,
|
||||||
|
unavailable_reason="removed",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
pa = Playlist(user_id=a.id, name="A")
|
||||||
|
pb = Playlist(user_id=b.id, name="B")
|
||||||
|
db.add_all([pa, pb])
|
||||||
|
db.flush()
|
||||||
|
db.add_all(
|
||||||
|
[
|
||||||
|
PlaylistItem(playlist_id=pa.id, video_id="v1", position=0),
|
||||||
|
PlaylistItem(playlist_id=pa.id, video_id="v2", position=1),
|
||||||
|
PlaylistItem(playlist_id=pb.id, video_id="v2", position=0),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
out = maintenance._recheck_flagged(db, _YT({}), now_utc())
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(Video, "v1") is None and db.get(Video, "v2") is None
|
||||||
|
assert db.execute(select(PlaylistItem)).scalars().all() == [] # the cascade
|
||||||
|
notes = {
|
||||||
|
n.user_id: n.data["count"]
|
||||||
|
for n in db.execute(select(Notification).where(Notification.type == "maintenance"))
|
||||||
|
.scalars()
|
||||||
|
.all()
|
||||||
|
}
|
||||||
|
assert notes == {a.id: 2, b.id: 1} # one per user, not one per video
|
||||||
|
assert out == {"checked": 2, "recovered": 0, "deleted": 2, "notified": 2}
|
||||||
|
|
||||||
|
|
||||||
|
def test_rolling_revalidation_flags_the_missing_and_keeps_a_live_stream(db, channel, monkeypatch):
|
||||||
|
monkeypatch.setattr(maintenance.sysconfig, "get_int", lambda db, key: 10)
|
||||||
|
monkeypatch.setattr(maintenance.quota, "backfill_reserve_reached", lambda: False)
|
||||||
|
db.add_all(
|
||||||
|
[
|
||||||
|
Video(id="ok", channel_id=channel.id),
|
||||||
|
Video(id="gone", channel_id=channel.id),
|
||||||
|
Video(id="live", channel_id=channel.id),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
yt = _YT({"ok": _item("ok"), "live": _item("live", liveBroadcastContent="live")})
|
||||||
|
out = maintenance._revalidate_rolling(db, yt, now_utc())
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(Video, "gone").unavailable_reason == "removed"
|
||||||
|
assert db.get(Video, "ok").unavailable_since is None
|
||||||
|
assert db.get(Video, "live").unavailable_since is None
|
||||||
|
assert out == {"checked": 3, "flagged": 1}
|
||||||
|
assert all(db.get(Video, v).last_checked_at is not None for v in ("ok", "gone", "live"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_run_asks_youtube_about_each_video_once(db, channel, monkeypatch, make_user):
|
||||||
|
# Phase 1 recovers v1; phase 2 must not take it as "not checked this run" and fetch it again.
|
||||||
|
monkeypatch.setattr(maintenance.sysconfig, "get_int", lambda db, key: 10)
|
||||||
|
monkeypatch.setattr(maintenance.quota, "backfill_reserve_reached", lambda: False)
|
||||||
|
yt = _YT({"v1": _item("v1"), "v2": _item("v2")})
|
||||||
|
|
||||||
|
class _Client:
|
||||||
|
def __init__(self, *_):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return yt
|
||||||
|
|
||||||
|
def __exit__(self, *_):
|
||||||
|
return False
|
||||||
|
|
||||||
|
monkeypatch.setattr(maintenance, "YouTubeClient", _Client)
|
||||||
|
monkeypatch.setattr(maintenance, "get_service_user", lambda db: make_user())
|
||||||
|
db.add_all(
|
||||||
|
[
|
||||||
|
Video(
|
||||||
|
id="v1",
|
||||||
|
channel_id=channel.id,
|
||||||
|
unavailable_since=now_utc(),
|
||||||
|
unavailable_reason="removed",
|
||||||
|
),
|
||||||
|
Video(id="v2", channel_id=channel.id),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
maintenance.run_maintenance(db)
|
||||||
|
asked = [vid for batch in yt.asked for vid in batch]
|
||||||
|
assert sorted(asked) == ["v1", "v2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_upcoming_premiere_counts_as_abandoned_only_when_it_never_went_live():
|
||||||
|
past = (now_utc() - timedelta(days=3)).isoformat()
|
||||||
|
soon = (now_utc() - timedelta(days=1)).isoformat()
|
||||||
|
|
||||||
|
def item(**live):
|
||||||
|
return {"snippet": {"liveBroadcastContent": "upcoming"}, "liveStreamingDetails": live}
|
||||||
|
|
||||||
|
assert maintenance._is_abandoned_upcoming(item(scheduledStartTime=past)) is True
|
||||||
|
assert maintenance._is_abandoned_upcoming(item(scheduledStartTime=soon)) is False
|
||||||
|
assert (
|
||||||
|
maintenance._is_abandoned_upcoming(item(scheduledStartTime=past, actualStartTime=past))
|
||||||
|
is False
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
maintenance._is_abandoned_upcoming({"snippet": {"liveBroadcastContent": "none"}}) is False
|
||||||
|
)
|
||||||
|
# …and an abandoned premiere has no grace period: the confirming sweep deletes it.
|
||||||
|
assert maintenance._grace_days("abandoned") == 0
|
||||||
@@ -97,7 +97,7 @@ def test_putting_a_removed_video_back_leaves_a_linked_list_clean(db, make_user,
|
|||||||
pl.synced_fingerprint = current_fingerprint(db, pl)
|
pl.synced_fingerprint = current_fingerprint(db, pl)
|
||||||
pl.dirty = False
|
pl.dirty = False
|
||||||
db.commit()
|
db.commit()
|
||||||
client.delete(f"/api/playlists/{pl.id}/items/b")
|
assert client.delete(f"/api/playlists/{pl.id}/items/b").status_code == 200
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
assert db.get(Playlist, pl.id).dirty is True
|
assert db.get(Playlist, pl.id).dirty is True
|
||||||
_add(client, pl.id, {"video_id": "b", "position": 1})
|
_add(client, pl.id, {"video_id": "b", "position": 1})
|
||||||
|
|||||||
@@ -12,8 +12,8 @@ import pytest
|
|||||||
|
|
||||||
from app.models import Channel, Playlist, PlaylistItem, Video
|
from app.models import Channel, Playlist, PlaylistItem, Video
|
||||||
from app.sync import playlists as pl_mod
|
from app.sync import playlists as pl_mod
|
||||||
|
from app.sync.playlist_merge import merge_plan
|
||||||
from app.sync.playlists import (
|
from app.sync.playlists import (
|
||||||
merge_plan,
|
|
||||||
merge_playlist,
|
merge_playlist,
|
||||||
parse_baseline,
|
parse_baseline,
|
||||||
parse_unavailable,
|
parse_unavailable,
|
||||||
|
|||||||
@@ -0,0 +1,154 @@
|
|||||||
|
"""routes/plex/admin — the connection test and the owner's watch-sync link (E32 S2: 37% covered).
|
||||||
|
|
||||||
|
The connection test is the admin's one early warning that the media mount or path map is wrong, and
|
||||||
|
it must not quote the server's URL back in a toast. The watch link runs the one-time "Plex is master"
|
||||||
|
import exactly once. Plex itself is a stub.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
import app.routes.plex.admin as admin_routes
|
||||||
|
from app import sysconfig
|
||||||
|
from app.models import PlexLink
|
||||||
|
from app.plex.client import PlexError, PlexNotConfigured
|
||||||
|
|
||||||
|
|
||||||
|
class _Plex:
|
||||||
|
def __init__(self, *, sections=None, leaf=None, fail=None):
|
||||||
|
self._sections = sections or []
|
||||||
|
self._leaf = leaf
|
||||||
|
self._fail = fail
|
||||||
|
self.section_items_calls: list[str] = []
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
if self._fail:
|
||||||
|
raise self._fail
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def server_info(self):
|
||||||
|
return {"friendlyName": "Basement", "version": "1.40"}
|
||||||
|
|
||||||
|
def sections(self):
|
||||||
|
return self._sections
|
||||||
|
|
||||||
|
def section_items(self, key, *_args):
|
||||||
|
self.section_items_calls.append(key)
|
||||||
|
return ([self._leaf] if self._leaf else []), 1
|
||||||
|
|
||||||
|
|
||||||
|
SECTIONS = [
|
||||||
|
{"key": 1, "title": "Movies", "type": "movie", "count": 10},
|
||||||
|
{"key": 2, "title": "Shows", "type": "show", "count": 3},
|
||||||
|
{"key": 3, "title": "Music", "type": "artist", "count": 99},
|
||||||
|
]
|
||||||
|
LEAF = {"Media": [{"Part": [{"file": "/data/movies/a.mkv"}]}]}
|
||||||
|
|
||||||
|
|
||||||
|
def _use(monkeypatch, plex):
|
||||||
|
monkeypatch.setattr(admin_routes, "PlexClient", lambda db: plex)
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_connection_test_lists_movie_and_show_sections_and_probes_one_file(
|
||||||
|
db, as_user, make_user, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
(tmp_path / "a.mkv").write_bytes(b"x")
|
||||||
|
sysconfig.set_value(db, "plex_path_map", f"/data/movies={tmp_path}")
|
||||||
|
sysconfig.set_value(db, "plex_libraries", "4")
|
||||||
|
db.commit()
|
||||||
|
# Only section 4 is enabled; the movie section listed first (1) must not be the one sampled.
|
||||||
|
other = {"key": 4, "title": "Kids", "type": "movie", "count": 2}
|
||||||
|
plex = _Plex(sections=[*SECTIONS, other], leaf=LEAF)
|
||||||
|
_use(monkeypatch, plex)
|
||||||
|
r = as_user(make_user(role="admin")).post("/api/plex/test")
|
||||||
|
assert r.status_code == 200, r.text
|
||||||
|
body = r.json()
|
||||||
|
assert body["server_name"] == "Basement"
|
||||||
|
assert [s["key"] for s in body["sections"]] == ["1", "2", "4"] # music is not ours
|
||||||
|
assert body["media_check"] == {
|
||||||
|
"checked": True,
|
||||||
|
"ok": True,
|
||||||
|
"plex_path": "/data/movies/a.mkv",
|
||||||
|
"local_path": str(tmp_path / "a.mkv"),
|
||||||
|
}
|
||||||
|
assert plex.section_items_calls == ["4"] # only an enabled movie section is sampled
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_wrong_mount_is_reported_and_an_empty_library_is_not_checked(
|
||||||
|
db, as_user, make_user, monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
sysconfig.set_value(db, "plex_path_map", f"/data/movies={tmp_path}")
|
||||||
|
db.commit()
|
||||||
|
client = as_user(make_user(role="admin"))
|
||||||
|
_use(monkeypatch, _Plex(sections=SECTIONS, leaf=LEAF)) # the file is not on the mount
|
||||||
|
check = client.post("/api/plex/test").json()["media_check"]
|
||||||
|
assert check["checked"] is True and check["ok"] is False
|
||||||
|
|
||||||
|
_use(monkeypatch, _Plex(sections=SECTIONS))
|
||||||
|
assert client.post("/api/plex/test").json()["media_check"] == {"checked": False}
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_failed_connection_says_so_without_quoting_the_server(
|
||||||
|
db, as_user, make_user, monkeypatch
|
||||||
|
):
|
||||||
|
client = as_user(make_user(role="admin"))
|
||||||
|
_use(monkeypatch, _Plex(fail=PlexNotConfigured()))
|
||||||
|
r = client.post("/api/plex/test")
|
||||||
|
assert r.status_code == 400
|
||||||
|
assert r.json()["detail"]["reason"] == "plex_not_configured"
|
||||||
|
|
||||||
|
_use(monkeypatch, _Plex(fail=PlexError("GET http://10.0.0.5:32400/secret refused")))
|
||||||
|
r = client.post("/api/plex/test")
|
||||||
|
assert r.status_code == 422
|
||||||
|
assert r.json()["detail"]["reason"] == "plex_unreachable"
|
||||||
|
assert "10.0.0.5" not in r.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_plex_admin_routes_refuse_a_regular_user(db, as_user, make_user, monkeypatch):
|
||||||
|
_use(monkeypatch, _Plex(sections=SECTIONS))
|
||||||
|
client = as_user(make_user())
|
||||||
|
for path in ("/api/plex/test", "/api/plex/sync", "/api/plex/watch/import"):
|
||||||
|
assert client.post(path).status_code == 403, path
|
||||||
|
assert client.post("/api/plex/watch/link", json={"enabled": True}).status_code == 403
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_watch_link_imports_once_and_needs_plex_on(db, as_user, make_user, monkeypatch):
|
||||||
|
imports: list[int] = []
|
||||||
|
|
||||||
|
def fake_import(db_, link):
|
||||||
|
imports.append(link.user_id)
|
||||||
|
link.initial_import_done = True
|
||||||
|
db_.commit()
|
||||||
|
return {"imported": 7}
|
||||||
|
|
||||||
|
monkeypatch.setattr(admin_routes.plex_watch, "import_owner_watch_state", fake_import)
|
||||||
|
owner = make_user(role="admin")
|
||||||
|
client = as_user(owner)
|
||||||
|
|
||||||
|
r = client.post("/api/plex/watch/link", json={"enabled": True})
|
||||||
|
assert r.status_code == 409 and r.json()["detail"]["reason"] == "plex_disabled"
|
||||||
|
assert client.post("/api/plex/watch/import").json()["detail"]["reason"] == "watch_sync_off"
|
||||||
|
|
||||||
|
sysconfig.set_value(db, "plex_enabled", True)
|
||||||
|
db.commit()
|
||||||
|
first = client.post("/api/plex/watch/link", json={"enabled": True})
|
||||||
|
assert first.status_code == 200
|
||||||
|
assert first.json()["import"] == {"imported": 7}
|
||||||
|
assert first.json()["linked"] and first.json()["sync_enabled"] and first.json()["uses_admin"]
|
||||||
|
|
||||||
|
assert client.post("/api/plex/watch/link", json={"enabled": False}).json()["import"] is None
|
||||||
|
again = client.post("/api/plex/watch/link", json={"enabled": True})
|
||||||
|
assert again.json()["import"] is None # the one-time import does not run twice
|
||||||
|
assert imports == [owner.id]
|
||||||
|
|
||||||
|
status = client.get("/api/plex/watch/link")
|
||||||
|
assert status.status_code == 200
|
||||||
|
assert status.json()["initial_import_done"] is True
|
||||||
|
links = db.execute(select(PlexLink).where(PlexLink.user_id == owner.id)).scalars().all()
|
||||||
|
assert len(links) == 1 # toggling never creates a second link
|
||||||
|
|
||||||
|
rerun = client.post("/api/plex/watch/import")
|
||||||
|
assert rerun.status_code == 200 and rerun.json()["import"] == {"imported": 7}
|
||||||
|
assert imports == [owner.id, owner.id]
|
||||||
@@ -0,0 +1,204 @@
|
|||||||
|
"""plex/client — the one door to the Plex server (E32 S2: 31% covered).
|
||||||
|
|
||||||
|
What matters here is what the callers cannot see: a failure must surface as PlexError (never as an
|
||||||
|
empty answer), the paged reads must stop where they should, and the write URIs must be well formed.
|
||||||
|
The network is an httpx MockTransport; the configuration is stubbed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.plex import client as plex_client
|
||||||
|
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def plex(monkeypatch):
|
||||||
|
"""A PlexClient on a stub server; set `routes[path] = handler-or-json` per test."""
|
||||||
|
routes: dict = {}
|
||||||
|
seen: list[httpx.Request] = []
|
||||||
|
|
||||||
|
def transport(request: httpx.Request) -> httpx.Response:
|
||||||
|
seen.append(request)
|
||||||
|
answer = routes.get(request.url.path)
|
||||||
|
if answer is None:
|
||||||
|
return httpx.Response(404)
|
||||||
|
if callable(answer):
|
||||||
|
return answer(request)
|
||||||
|
return httpx.Response(200, json={"MediaContainer": answer})
|
||||||
|
|
||||||
|
real = httpx.Client
|
||||||
|
monkeypatch.setattr(
|
||||||
|
plex_client.httpx,
|
||||||
|
"Client",
|
||||||
|
lambda **kw: real(transport=httpx.MockTransport(transport), **kw),
|
||||||
|
)
|
||||||
|
conf = {"plex_server_url": "http://plex.test:32400/", "plex_server_token": "tok"}
|
||||||
|
monkeypatch.setattr(plex_client.sysconfig, "get_str", lambda db, key: conf[key])
|
||||||
|
c = PlexClient(None) # type: ignore[arg-type] # the stubbed config never reads the session
|
||||||
|
c.routes, c.seen = routes, seen # type: ignore[attr-defined]
|
||||||
|
yield c
|
||||||
|
c.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_unconfigured_server_is_refused_up_front(monkeypatch):
|
||||||
|
monkeypatch.setattr(plex_client.sysconfig, "get_str", lambda db, key: "")
|
||||||
|
with pytest.raises(PlexNotConfigured):
|
||||||
|
PlexClient(None) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
|
||||||
|
def test_reads_carry_the_token_and_unwrap_the_container(plex):
|
||||||
|
plex.routes["/library/sections"] = {"Directory": [{"key": "1", "type": "movie"}]}
|
||||||
|
assert plex.sections() == [{"key": "1", "type": "movie"}]
|
||||||
|
req = plex.seen[-1]
|
||||||
|
assert str(req.url) == "http://plex.test:32400/library/sections"
|
||||||
|
assert req.headers["x-plex-token"] == "tok"
|
||||||
|
assert req.headers["accept"] == "application/json"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"answer",
|
||||||
|
[
|
||||||
|
lambda r: httpx.Response(500),
|
||||||
|
lambda r: httpx.Response(200, text="<html>login</html>"),
|
||||||
|
lambda r: (_ for _ in ()).throw(httpx.ConnectError("down", request=r)),
|
||||||
|
],
|
||||||
|
ids=["http-500", "not-json", "unreachable"],
|
||||||
|
)
|
||||||
|
def test_a_failed_read_is_a_plex_error_never_an_empty_answer(plex, answer):
|
||||||
|
plex.routes["/library/sections"] = answer
|
||||||
|
with pytest.raises(PlexError):
|
||||||
|
plex.sections()
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_section_page_asks_for_guids_and_reports_the_server_total(plex):
|
||||||
|
plex.routes["/library/sections/7/all"] = {"Metadata": [{"ratingKey": "1"}], "totalSize": 250}
|
||||||
|
items, total = plex.section_items("7", 1, 200, 50)
|
||||||
|
assert items == [{"ratingKey": "1"}] and total == 250
|
||||||
|
params = plex.seen[-1].url.params
|
||||||
|
assert params["includeGuids"] == "1" and params["type"] == "1"
|
||||||
|
assert params["X-Plex-Container-Start"] == "200" and params["X-Plex-Container-Size"] == "50"
|
||||||
|
|
||||||
|
# Measured on the real server (E32 review): a paged request carries `totalSize` even when the
|
||||||
|
# page is empty, past the end, or the section holds nothing — `size: 0, totalSize: 0`.
|
||||||
|
plex.routes["/library/sections/7/all"] = {"Metadata": [], "size": 0, "totalSize": 0}
|
||||||
|
assert plex.section_items("7", 2) == ([], 0)
|
||||||
|
# No total at all is not "0": the caller must not read it as a complete, empty section.
|
||||||
|
plex.routes["/library/sections/7/all"] = {"Metadata": []}
|
||||||
|
assert plex.section_items("7", 2) == ([], None)
|
||||||
|
# …and `size` is THIS page's count, never the section's: one full page is not the whole section.
|
||||||
|
plex.routes["/library/sections/7/all"] = {"Metadata": [{"ratingKey": "1"}], "size": 1}
|
||||||
|
assert plex.section_items("7", 2) == ([{"ratingKey": "1"}], None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_related_flattens_the_hubs_and_degrades_to_empty(plex):
|
||||||
|
plex.routes["/library/metadata/5/related"] = {
|
||||||
|
"Hub": [{"Metadata": [{"ratingKey": "a"}]}, {"Metadata": [{"ratingKey": "b"}]}, {}]
|
||||||
|
}
|
||||||
|
assert [m["ratingKey"] for m in plex.related("5")] == ["a", "b"]
|
||||||
|
plex.routes["/library/metadata/5/related"] = lambda r: httpx.Response(500)
|
||||||
|
assert plex.related("5") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_metadata_is_the_first_item_or_none(plex):
|
||||||
|
plex.routes["/library/metadata/9"] = {"Metadata": [{"ratingKey": "9", "title": "X"}]}
|
||||||
|
assert plex.metadata("9", markers=True) == {"ratingKey": "9", "title": "X"}
|
||||||
|
assert plex.seen[-1].url.params["includeMarkers"] == "1"
|
||||||
|
plex.routes["/library/metadata/9"] = {}
|
||||||
|
assert plex.metadata("9") is None
|
||||||
|
|
||||||
|
|
||||||
|
def _history(rows_by_start: dict[int, list[dict]]):
|
||||||
|
def answer(request: httpx.Request) -> httpx.Response:
|
||||||
|
start = int(request.url.params["X-Plex-Container-Start"])
|
||||||
|
return httpx.Response(
|
||||||
|
200, json={"MediaContainer": {"Metadata": rows_by_start.get(start, [])}}
|
||||||
|
)
|
||||||
|
|
||||||
|
return answer
|
||||||
|
|
||||||
|
|
||||||
|
def test_watch_history_pages_until_the_cutoff_and_keeps_only_the_owner(plex):
|
||||||
|
plex.routes["/status/sessions/history/all"] = _history(
|
||||||
|
{
|
||||||
|
0: [
|
||||||
|
{"ratingKey": "1", "viewedAt": 500, "accountID": 2},
|
||||||
|
{"ratingKey": "2", "viewedAt": 400, "accountID": 1},
|
||||||
|
],
|
||||||
|
2: [
|
||||||
|
{"ratingKey": "3", "viewedAt": 300, "accountID": 1},
|
||||||
|
{"ratingKey": "4", "viewedAt": 100, "accountID": 1},
|
||||||
|
],
|
||||||
|
4: [{"ratingKey": "never", "viewedAt": 50, "accountID": 1}],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
rows = plex.watch_history(min_viewed_at=200, account_id=1, size=2)
|
||||||
|
assert [r["ratingKey"] for r in rows] == ["2", "3"] # page 2 reached, the cutoff stops it
|
||||||
|
assert [int(r.url.params["X-Plex-Container-Start"]) for r in plex.seen] == [0, 2]
|
||||||
|
|
||||||
|
|
||||||
|
def test_watch_history_stops_on_a_short_page_and_at_the_page_cap(plex):
|
||||||
|
plex.routes["/status/sessions/history/all"] = _history({0: [{"ratingKey": "1", "viewedAt": 9}]})
|
||||||
|
assert [r["ratingKey"] for r in plex.watch_history(size=2)] == ["1"]
|
||||||
|
assert len(plex.seen) == 1
|
||||||
|
|
||||||
|
full = [{"ratingKey": "r", "viewedAt": 9}, {"ratingKey": "r", "viewedAt": 9}]
|
||||||
|
plex.routes["/status/sessions/history/all"] = lambda r: httpx.Response(
|
||||||
|
200, json={"MediaContainer": {"Metadata": full}}
|
||||||
|
)
|
||||||
|
plex.seen.clear()
|
||||||
|
assert len(plex.watch_history(size=2, max_pages=3)) == 6
|
||||||
|
assert len(plex.seen) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_collection_is_created_with_a_well_formed_uri(plex):
|
||||||
|
plex.routes["/identity"] = {"machineIdentifier": "abc"}
|
||||||
|
plex.routes["/library/collections"] = {"Metadata": [{"ratingKey": "c1"}]}
|
||||||
|
assert plex.create_collection("3", "Best of: 90s", "42") == {"ratingKey": "c1"}
|
||||||
|
params = plex.seen[-1].url.params
|
||||||
|
assert plex.seen[-1].method == "POST"
|
||||||
|
assert params["title"] == "Best of: 90s" and params["sectionId"] == "3"
|
||||||
|
assert params["uri"] == "server://abc/com.plexapp.plugins.library/library/metadata/42"
|
||||||
|
|
||||||
|
plex.routes["/library/collections"] = {}
|
||||||
|
with pytest.raises(PlexError):
|
||||||
|
plex.create_collection("3", "Empty", "42")
|
||||||
|
assert [r.url.path for r in plex.seen].count("/identity") == 1 # cached
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_server_without_a_machine_id_refuses_the_write(plex):
|
||||||
|
"""No machineIdentifier used to become `server:///…` in the write URI — a malformed request
|
||||||
|
Plex answers in its own way. It is refused here, named (E32 S2)."""
|
||||||
|
plex.routes["/identity"] = {}
|
||||||
|
plex.routes["/library/collections"] = {"Metadata": [{"ratingKey": "c1"}]}
|
||||||
|
with pytest.raises(PlexError, match="machine"):
|
||||||
|
plex.create_collection("3", "T", "42")
|
||||||
|
assert all(r.url.path == "/identity" for r in plex.seen)
|
||||||
|
|
||||||
|
|
||||||
|
def test_watch_state_writes_target_the_library_item(plex):
|
||||||
|
for path in ("/:/scrobble", "/:/unscrobble", "/:/timeline"):
|
||||||
|
plex.routes[path] = lambda r: httpx.Response(200) # Plex answers with an empty body
|
||||||
|
plex.scrobble("42")
|
||||||
|
assert plex.seen[-1].url.params["key"] == "42"
|
||||||
|
plex.unscrobble("42")
|
||||||
|
assert plex.seen[-1].url.path == "/:/unscrobble"
|
||||||
|
plex.set_timeline("42", 61_500.7, 120_000) # type: ignore[arg-type]
|
||||||
|
p = plex.seen[-1].url.params
|
||||||
|
assert p["key"] == "/library/metadata/42" and p["time"] == "61500" and p["state"] == "stopped"
|
||||||
|
plex.routes["/:/scrobble"] = lambda r: httpx.Response(401)
|
||||||
|
with pytest.raises(PlexError):
|
||||||
|
plex.scrobble("42")
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_image_is_resized_by_plex_when_a_box_is_given(plex):
|
||||||
|
plex.routes["/photo/:/transcode"] = lambda r: httpx.Response(
|
||||||
|
200, content=b"small", headers={"Content-Type": "image/webp"}
|
||||||
|
)
|
||||||
|
plex.routes["/library/metadata/1/thumb/2"] = lambda r: httpx.Response(200, content=b"full")
|
||||||
|
assert plex.image_bytes("/library/metadata/1/thumb/2", 300, 450) == (b"small", "image/webp")
|
||||||
|
assert plex.seen[-1].url.params["url"] == "/library/metadata/1/thumb/2"
|
||||||
|
body, ctype = plex.image_bytes("/library/metadata/1/thumb/2")
|
||||||
|
assert body == b"full" and ctype == "image/jpeg"
|
||||||
|
with pytest.raises(PlexError):
|
||||||
|
plex.raw_get("/library/streams/404")
|
||||||
@@ -67,7 +67,7 @@ def test_each_variant_is_cut_at_its_own_size(db, make_user, monkeypatch, tmp_pat
|
|||||||
]
|
]
|
||||||
# Each cut is cached under its own name: asking again reaches Plex no more.
|
# Each cut is cached under its own name: asking again reaches Plex no more.
|
||||||
for variant in ("thumb", "thumb_sm", "art", "art_sm"):
|
for variant in ("thumb", "thumb_sm", "art", "art_sm"):
|
||||||
client.get(f"/api/plex/image/pm1?variant={variant}")
|
assert client.get(f"/api/plex/image/pm1?variant={variant}").status_code == 200
|
||||||
assert len(_FakePlex.calls) == 4
|
assert len(_FakePlex.calls) == 4
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -16,18 +16,21 @@ from app import sysconfig
|
|||||||
from app.db import engine
|
from app.db import engine
|
||||||
from app.models import PlexItem, PlexLibrary, PlexShow
|
from app.models import PlexItem, PlexLibrary, PlexShow
|
||||||
from app.plex.genres import hu_genre, hu_genres
|
from app.plex.genres import hu_genre, hu_genres
|
||||||
from app.plex.sync import _apply_item, _canonical_guid, _sync_overlay
|
from app.plex.overlay import sync_overlay
|
||||||
|
from app.plex.sync import _apply_item, _canonical_guid
|
||||||
from app.plex.query import content_libs
|
from app.plex.query import content_libs
|
||||||
|
|
||||||
|
|
||||||
class _StubPlex:
|
class _StubPlex:
|
||||||
"""Minimal PlexClient stand-in: _sync_overlay only calls section_items via _paginate."""
|
"""Minimal PlexClient stand-in: sync_overlay only calls section_items via _paginate. `total`
|
||||||
|
over the item count is a listing cut short (the server said more than it handed over)."""
|
||||||
|
|
||||||
def __init__(self, items: list[dict]):
|
def __init__(self, items: list[dict], total: int | None = None):
|
||||||
self._items = items
|
self._items = items
|
||||||
|
self._total = len(items) if total is None else total
|
||||||
|
|
||||||
def section_items(self, key, item_type, start, size):
|
def section_items(self, key, item_type, start, size):
|
||||||
return self._items[start : start + size], len(self._items)
|
return self._items[start : start + size], self._total
|
||||||
|
|
||||||
|
|
||||||
def _hu_movie(rk: str, title: str, summary: str, genres: list[str], guid: str | None) -> dict:
|
def _hu_movie(rk: str, title: str, summary: str, genres: list[str], guid: str | None) -> dict:
|
||||||
@@ -71,7 +74,7 @@ def test_sync_overlay_writes_hu_onto_matched_en_rows(db):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
stats = {"overlaid": 0}
|
stats = {"overlaid": 0}
|
||||||
_sync_overlay(db, stub, hu_lib, en_lib.id, stats)
|
sync_overlay(db, stub, hu_lib, en_lib.id, stats)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
db.refresh(matched)
|
db.refresh(matched)
|
||||||
@@ -185,13 +188,51 @@ def test_overlay_reconcile_clears_stale_hu_when_twin_vanishes(db):
|
|||||||
)
|
)
|
||||||
db.add(row)
|
db.add(row)
|
||||||
db.commit()
|
db.commit()
|
||||||
# The HU section no longer lists this film (empty overlay).
|
# The HU section, read in full, no longer lists this film (only another one).
|
||||||
_sync_overlay(db, _StubPlex([]), hu_lib, en_lib.id, {"overlaid": 0})
|
other = _hu_movie("h9", "Más", "más", [], "tmdb://99")
|
||||||
|
sync_overlay(db, _StubPlex([other]), hu_lib, en_lib.id, {"overlaid": 0})
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(row)
|
db.refresh(row)
|
||||||
assert row.title_hu is None and row.summary_hu is None and row.genres_hu is None
|
assert row.title_hu is None and row.summary_hu is None and row.genres_hu is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_listing_cut_short_keeps_the_hungarian_text_it_did_not_reach(db):
|
||||||
|
"""E32 S2: the reconcile clears HU text only after a COMPLETE listing. A walk a Plex hiccup cut
|
||||||
|
short (an empty page before the server's total), or an empty section, says nothing about the
|
||||||
|
titles it did not reach — clearing them dropped their Hungarian text until the next full sync."""
|
||||||
|
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||||
|
hu_lib = PlexLibrary(plex_key="9", title="Filmek", kind="movie", metadata_lang="hu")
|
||||||
|
db.add_all([en_lib, hu_lib])
|
||||||
|
db.flush()
|
||||||
|
hu_lib.overlay_of = en_lib.id
|
||||||
|
kept = PlexItem(
|
||||||
|
rating_key="m1",
|
||||||
|
library_id=en_lib.id,
|
||||||
|
kind="movie",
|
||||||
|
title="Heat",
|
||||||
|
guid="tmdb:1",
|
||||||
|
title_hu="Szemtől szemben",
|
||||||
|
summary_hu="régi",
|
||||||
|
genres_hu=["Akció"],
|
||||||
|
)
|
||||||
|
fresh = PlexItem(rating_key="m2", library_id=en_lib.id, kind="movie", title="Up", guid="tmdb:2")
|
||||||
|
db.add_all([kept, fresh])
|
||||||
|
db.commit()
|
||||||
|
stats: dict = {}
|
||||||
|
reached = _hu_movie("h2", "Fel", "fel", [], "tmdb://2")
|
||||||
|
sync_overlay(db, _StubPlex([reached], total=2), hu_lib, en_lib.id, stats)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(kept)
|
||||||
|
db.refresh(fresh)
|
||||||
|
assert kept.title_hu == "Szemtől szemben" and kept.summary_hu == "régi"
|
||||||
|
assert fresh.title_hu == "Fel" # what the listing did reach is still applied
|
||||||
|
assert stats["overlay_incomplete"] == 1
|
||||||
|
sync_overlay(db, _StubPlex([]), hu_lib, en_lib.id, {}) # an empty section: the same answer
|
||||||
|
db.commit()
|
||||||
|
db.refresh(kept)
|
||||||
|
assert kept.title_hu == "Szemtől szemben"
|
||||||
|
|
||||||
|
|
||||||
def test_apply_item_keeps_prior_guid_when_listing_lacks_guid():
|
def test_apply_item_keeps_prior_guid_when_listing_lacks_guid():
|
||||||
row = PlexItem(rating_key="m1", kind="movie", guid="tmdb:1")
|
row = PlexItem(rating_key="m1", kind="movie", guid="tmdb:1")
|
||||||
_apply_item(row, lib_id=7, kind="movie", meta={"title": "Heat"}) # no Guid array
|
_apply_item(row, lib_id=7, kind="movie", meta={"title": "Heat"}) # no Guid array
|
||||||
@@ -225,7 +266,7 @@ def test_overlay_applies_to_all_editions_sharing_a_guid(db):
|
|||||||
]
|
]
|
||||||
)
|
)
|
||||||
stats = {"overlaid": 0}
|
stats = {"overlaid": 0}
|
||||||
_sync_overlay(db, stub, hu_lib, en_lib.id, stats)
|
sync_overlay(db, stub, hu_lib, en_lib.id, stats)
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(ed1)
|
db.refresh(ed1)
|
||||||
db.refresh(ed2)
|
db.refresh(ed2)
|
||||||
@@ -291,7 +332,7 @@ def test_a_show_overlay_writes_hungarian_episode_titles_and_summaries(db):
|
|||||||
],
|
],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
_sync_overlay(db, stub, hu_lib, en_lib.id, {})
|
sync_overlay(db, stub, hu_lib, en_lib.id, {})
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(ep)
|
db.refresh(ep)
|
||||||
db.refresh(show)
|
db.refresh(show)
|
||||||
@@ -328,7 +369,7 @@ def test_the_overlay_gives_every_row_one_hungarian_word_per_genre(db):
|
|||||||
"h1", "Szemtől szemben", "HU.", ["Action", "Bűnügyi", "Drama", "Dráma"], "tmdb://1"
|
"h1", "Szemtől szemben", "HU.", ["Action", "Bűnügyi", "Drama", "Dráma"], "tmdb://1"
|
||||||
)
|
)
|
||||||
meta["tagline"] = "Magyar mottó."
|
meta["tagline"] = "Magyar mottó."
|
||||||
_sync_overlay(db, _StubPlex([meta]), hu_lib, en_lib.id, {})
|
sync_overlay(db, _StubPlex([meta]), hu_lib, en_lib.id, {})
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(twin)
|
db.refresh(twin)
|
||||||
db.refresh(lone)
|
db.refresh(lone)
|
||||||
@@ -353,7 +394,7 @@ def test_a_twin_without_genres_keeps_the_rows_own_hungarian_list(db):
|
|||||||
)
|
)
|
||||||
db.add(row)
|
db.add(row)
|
||||||
db.commit()
|
db.commit()
|
||||||
_sync_overlay(
|
sync_overlay(
|
||||||
db,
|
db,
|
||||||
_StubPlex([_hu_movie("h1", "Szemtől szemben", "HU.", [], "tmdb://1")]),
|
_StubPlex([_hu_movie("h1", "Szemtől szemben", "HU.", [], "tmdb://1")]),
|
||||||
hu_lib,
|
hu_lib,
|
||||||
|
|||||||
@@ -0,0 +1,82 @@
|
|||||||
|
"""plex/paths — the Plex-side path → local mount translation (E32 S2: 23% covered, and a wrong answer
|
||||||
|
here reads as "media file not found" for a whole library at once, with nothing else failing).
|
||||||
|
|
||||||
|
Pure logic: the admin map is read through sysconfig, stubbed per test.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.plex import paths
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def path_map(monkeypatch):
|
||||||
|
def use(raw: str) -> None:
|
||||||
|
monkeypatch.setattr(paths.sysconfig, "get_str", lambda db, key: raw)
|
||||||
|
|
||||||
|
return use
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_map_parses_both_separators_drops_junk_and_puts_the_longest_prefix_first():
|
||||||
|
raw = " /data=/mnt/a ,\n/data/movies = /mnt/m\nnot-a-pair,=/x,/y=, /data/tv=/mnt/tv "
|
||||||
|
assert paths.parse_map(raw) == [
|
||||||
|
("/data/movies", "/mnt/m"),
|
||||||
|
("/data/tv", "/mnt/tv"),
|
||||||
|
("/data", "/mnt/a"),
|
||||||
|
]
|
||||||
|
assert paths.parse_map("") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_longest_matching_prefix_wins(path_map):
|
||||||
|
path_map("/data=/mnt/a,/data/movies=/mnt/m")
|
||||||
|
assert paths.map_path(None, "/data/movies/Foo.mkv") == str(Path("/mnt/m") / "Foo.mkv")
|
||||||
|
assert paths.map_path(None, "/data/tv/Bar.mkv") == str(Path("/mnt/a") / "tv/Bar.mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_prefix_matches_only_at_a_path_boundary(path_map):
|
||||||
|
# "/data/movies2" shares the characters of "/data/movies" but is another folder.
|
||||||
|
path_map("/data/movies=/mnt/m")
|
||||||
|
assert paths.map_path(None, "/data/movies2/Foo.mkv") == "/data/movies2/Foo.mkv"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_prefix_itself_and_a_trailing_slash_in_the_map(path_map):
|
||||||
|
path_map("/data/movies/=/mnt/m")
|
||||||
|
assert paths.map_path(None, "/data/movies/Foo.mkv") == str(Path("/mnt/m") / "Foo.mkv")
|
||||||
|
path_map("/data/movies=/mnt/m")
|
||||||
|
assert paths.map_path(None, "/data/movies") == "/mnt/m"
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_map_leaves_the_path_as_plex_sees_it(path_map):
|
||||||
|
path_map("")
|
||||||
|
assert paths.map_path(None, "/data/movies/Foo.mkv") == "/data/movies/Foo.mkv"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_local_file_must_exist(path_map, tmp_path):
|
||||||
|
(tmp_path / "movies").mkdir()
|
||||||
|
(tmp_path / "movies" / "Foo.mkv").write_bytes(b"x")
|
||||||
|
path_map(f"/data={tmp_path}")
|
||||||
|
assert (
|
||||||
|
paths.local_media_path(None, "/data/movies/Foo.mkv")
|
||||||
|
== (tmp_path / "movies" / "Foo.mkv").resolve()
|
||||||
|
)
|
||||||
|
assert paths.local_media_path(None, "/data/movies/Missing.mkv") is None
|
||||||
|
assert paths.local_media_path(None, "/data/movies") is None # a folder is not a file
|
||||||
|
assert paths.local_media_path(None, None) is None
|
||||||
|
assert paths.local_media_path(None, "") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_path_that_cannot_be_resolved_is_logged_and_answered_none(path_map, monkeypatch, caplog):
|
||||||
|
path_map("")
|
||||||
|
|
||||||
|
def broken(self, strict=False):
|
||||||
|
raise OSError("dead mount")
|
||||||
|
|
||||||
|
monkeypatch.setattr(paths.Path, "resolve", broken)
|
||||||
|
assert paths.local_media_path(None, "/data/movies/Foo.mkv") is None
|
||||||
|
rec = next(
|
||||||
|
r for r in caplog.records if getattr(r, "event", None) == "plex.paths.resolve_failed"
|
||||||
|
)
|
||||||
|
# The title stays out of the log line: the path carries it.
|
||||||
|
assert "Foo" not in rec.getMessage() and rec.attrs == {"reason": "OSError"}
|
||||||
@@ -105,8 +105,11 @@ def test_remove_then_put_back_is_no_change(db, make_user, as_user):
|
|||||||
user = make_user()
|
user = make_user()
|
||||||
pl = _seed(db, user, ["a", "b", "c", "d"], extra=())
|
pl = _seed(db, user, ["a", "b", "c", "d"], extra=())
|
||||||
client = as_user(user)
|
client = as_user(user)
|
||||||
client.post(
|
assert (
|
||||||
f"/api/plex/playlists/{pl.id}/items/remove-bulk", json={"item_rating_keys": ["b", "d"]}
|
client.post(
|
||||||
|
f"/api/plex/playlists/{pl.id}/items/remove-bulk", json={"item_rating_keys": ["b", "d"]}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
assert _order(db, pl.id)[0] == ["a", "c"]
|
assert _order(db, pl.id)[0] == ["a", "c"]
|
||||||
_add(client, pl.id, ["b", "d"], [1, 3])
|
_add(client, pl.id, ["b", "d"], [1, 3])
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ Since the sweep stopped being awaited at startup it also runs CONCURRENTLY with
|
|||||||
second half of this file pins what keeps it off a LIVE session's segments."""
|
second half of this file pins what keeps it off a LIVE session's segments."""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -830,6 +831,46 @@ class TestProbeVideoKind:
|
|||||||
assert stream._probe_video_kind(Path("/x")) == stream._VK_HWSAFE
|
assert stream._probe_video_kind(Path("/x")) == stream._VK_HWSAFE
|
||||||
|
|
||||||
|
|
||||||
|
class TestProbeExitStatus:
|
||||||
|
"""A probe whose ffprobe EXITS NON-ZERO is a failed probe, not an answer (E32). Each probe used to
|
||||||
|
read stdout without looking at the exit status, so a refusing ffprobe silently picked the fallback
|
||||||
|
— HW-safe, single audio, the requested offset — and skipped the warning its except branch promises.
|
||||||
|
Drives the REAL subprocess.run with the probe's own kwargs against a stand-in command that prints a
|
||||||
|
plausible answer and then fails, so the test fails whenever the exit status is ignored."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _failing_ffprobe(monkeypatch, stdout: str):
|
||||||
|
real_run = stream.subprocess.run
|
||||||
|
script = f"import sys; sys.stdout.write({stdout!r}); sys.exit(1)"
|
||||||
|
monkeypatch.setattr(
|
||||||
|
stream.subprocess,
|
||||||
|
"run",
|
||||||
|
lambda _cmd, **kw: real_run([sys.executable, "-c", script], **kw),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _events(caplog) -> list[str]:
|
||||||
|
return [getattr(r, "event", "") for r in caplog.records]
|
||||||
|
|
||||||
|
def test_video_kind(self, monkeypatch, caplog):
|
||||||
|
self._failing_ffprobe(monkeypatch, "pix_fmt=yuv420p10le\ncolor_transfer=smpte2084\n")
|
||||||
|
with caplog.at_level("INFO"):
|
||||||
|
assert stream._probe_video_kind(Path("/x")) == stream._VK_HWSAFE
|
||||||
|
assert "plex.stream.probe_kind_failed" in self._events(caplog)
|
||||||
|
|
||||||
|
def test_audio_langs(self, monkeypatch, caplog):
|
||||||
|
self._failing_ffprobe(monkeypatch, "hun\neng\n")
|
||||||
|
with caplog.at_level("INFO"):
|
||||||
|
assert stream._probe_audio_langs(Path("/x")) == []
|
||||||
|
assert "plex.stream.probe_audio_failed" in self._events(caplog)
|
||||||
|
|
||||||
|
def test_first_pts(self, monkeypatch, caplog):
|
||||||
|
self._failing_ffprobe(monkeypatch, "12.5\n")
|
||||||
|
with caplog.at_level("INFO"):
|
||||||
|
assert stream._probe_first_pts(Path("/x")) is None
|
||||||
|
assert "plex.stream.first_pts_probe_failed" in self._events(caplog)
|
||||||
|
|
||||||
|
|
||||||
class TestOpenclPresent:
|
class TestOpenclPresent:
|
||||||
"""The OpenCL-runtime probe (_opencl_present) — the precondition for the HW HDR tonemap graph. Drives
|
"""The OpenCL-runtime probe (_opencl_present) — the precondition for the HW HDR tonemap graph. Drives
|
||||||
the REAL glob against a temp ICD dir instead of monkeypatching the function, so a regression to the
|
the REAL glob against a temp ICD dir instead of monkeypatching the function, so a regression to the
|
||||||
|
|||||||
@@ -208,7 +208,7 @@ def test_an_unshowable_track_says_why_and_maps_to_a_status(client, monkeypatch):
|
|||||||
raise subtitles.SubtitleUnavailable("bitmap")
|
raise subtitles.SubtitleUnavailable("bitmap")
|
||||||
|
|
||||||
monkeypatch.setattr(subtitles, "_extract_and_publish", _bitmap)
|
monkeypatch.setattr(subtitles, "_extract_and_publish", _bitmap)
|
||||||
client.get("/api/plex/subtitle/98614/0/status")
|
assert client.get("/api/plex/subtitle/98614/0/status").status_code == 200
|
||||||
pool.run_all()
|
pool.run_all()
|
||||||
st = client.get("/api/plex/subtitle/98614/0/status").json()
|
st = client.get("/api/plex/subtitle/98614/0/status").json()
|
||||||
assert st["state"] == "failed" and st["code"] == "bitmap"
|
assert st["state"] == "failed" and st["code"] == "bitmap"
|
||||||
|
|||||||
@@ -0,0 +1,277 @@
|
|||||||
|
"""The daily full watch reconcile (`run_plex_watch_reconcile`) — the one pass that DELETES watch state:
|
||||||
|
a mirrored row Plex no longer has a record for is read as "Plex un-watched it" and cleared. E32 S2
|
||||||
|
found it trusting a scan a Plex hiccup had cut short; these hold what it may and may not clear.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app import sysconfig
|
||||||
|
from app.models import PlexItem, PlexLibrary, PlexLink, PlexState, User
|
||||||
|
from app.plex import watch_sync
|
||||||
|
|
||||||
|
|
||||||
|
class _FakePlex:
|
||||||
|
"""A PlexClient stand-in: one movie section whose listing reports `total` (more than it hands
|
||||||
|
over = a walk cut short), plus any `more` sections as `(type, leaves, total)` — listed AFTER
|
||||||
|
the movies — and a record of every write the reconcile sends back to Plex."""
|
||||||
|
|
||||||
|
def __init__(self, leaves: list[dict], total: int | None = None, more=()):
|
||||||
|
self.listings = {"1": ("movie", leaves, len(leaves) if total is None else total)}
|
||||||
|
for i, (stype, extra, extra_total) in enumerate(more, start=2):
|
||||||
|
self.listings[str(i)] = (stype, extra, extra_total)
|
||||||
|
self.calls: list[tuple] = []
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def sections(self):
|
||||||
|
return [{"type": stype, "key": key} for key, (stype, _l, _t) in self.listings.items()]
|
||||||
|
|
||||||
|
def section_items(self, key, item_type, start, size):
|
||||||
|
_stype, leaves, total = self.listings[key]
|
||||||
|
return leaves[start : start + size], total
|
||||||
|
|
||||||
|
def scrobble(self, rk):
|
||||||
|
self.calls.append(("scrobble", rk))
|
||||||
|
|
||||||
|
def unscrobble(self, rk):
|
||||||
|
self.calls.append(("unscrobble", rk))
|
||||||
|
|
||||||
|
def set_timeline(self, rk, time_ms, duration_ms):
|
||||||
|
self.calls.append(("timeline", rk, time_ms))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def world(db):
|
||||||
|
"""Five mirrored movies and the owner's link; each test sets the local states it needs."""
|
||||||
|
sysconfig.set_value(db, "plex_enabled", True)
|
||||||
|
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||||
|
user = User(email="reconcile@example.com")
|
||||||
|
db.add_all([lib, user])
|
||||||
|
db.flush()
|
||||||
|
items = {}
|
||||||
|
for rk in ("m1", "m2", "m3", "m4", "m5"):
|
||||||
|
items[rk] = PlexItem(
|
||||||
|
rating_key=rk, library_id=lib.id, kind="movie", title=rk, duration_s=6000
|
||||||
|
)
|
||||||
|
db.add(items[rk])
|
||||||
|
db.add(
|
||||||
|
PlexLink(
|
||||||
|
user_id=user.id,
|
||||||
|
uses_admin=True,
|
||||||
|
sync_enabled=True,
|
||||||
|
initial_import_done=True,
|
||||||
|
plex_account_id=1,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
def state(rk, **kw):
|
||||||
|
db.add(PlexState(user_id=user.id, item_id=items[rk].id, **kw))
|
||||||
|
|
||||||
|
def rows():
|
||||||
|
db.expire_all()
|
||||||
|
by_id = {i.id: rk for rk, i in items.items()}
|
||||||
|
return {
|
||||||
|
by_id[s.item_id]: (s.status, s.position_seconds, s.synced_to_plex)
|
||||||
|
for s in db.execute(select(PlexState).where(PlexState.user_id == user.id)).scalars()
|
||||||
|
if s.item_id in by_id # the five movies; a test's own extra items it reads itself
|
||||||
|
}
|
||||||
|
|
||||||
|
return state, rows
|
||||||
|
|
||||||
|
|
||||||
|
def _run(db, monkeypatch, plex):
|
||||||
|
monkeypatch.setattr(watch_sync, "PlexClient", lambda _db: plex)
|
||||||
|
return watch_sync.run_plex_watch_reconcile(db)
|
||||||
|
|
||||||
|
|
||||||
|
def _local_states(state):
|
||||||
|
state("m1", status="watched", synced_to_plex=True) # mirrored; Plex has no record now
|
||||||
|
state("m2", status="watched", synced_to_plex=False) # local change that never reached Plex
|
||||||
|
state("m3", status="hidden", synced_to_plex=True) # Siftlode-only, never cleared
|
||||||
|
state("m5", status="new", position_seconds=300, synced_to_plex=True) # Plex says otherwise
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_complete_scan_clears_what_plex_unwatched_and_keeps_the_rest(db, monkeypatch, world):
|
||||||
|
state, rows = world
|
||||||
|
_local_states(state)
|
||||||
|
db.commit()
|
||||||
|
plex = _FakePlex(
|
||||||
|
[
|
||||||
|
{"ratingKey": "m3"}, # listed, no watch signal
|
||||||
|
{"ratingKey": "m4", "viewCount": 1, "lastViewedAt": 1_700_000_000},
|
||||||
|
{"ratingKey": "m5", "viewOffset": 120_000, "lastViewedAt": 1_700_000_000},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
out = _run(db, monkeypatch, plex)
|
||||||
|
assert rows() == {
|
||||||
|
"m2": ("watched", 0, True), # re-pushed, then settled
|
||||||
|
"m3": ("hidden", 0, True),
|
||||||
|
"m4": ("watched", 0, True), # pulled in
|
||||||
|
"m5": ("new", 120, True), # Plex's position wins
|
||||||
|
}
|
||||||
|
assert ("scrobble", "m2") in plex.calls
|
||||||
|
assert out["cleared"] == 1 and out["repushed"] == 1 and "incomplete" not in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_scan_cut_short_clears_nothing(db, monkeypatch, world):
|
||||||
|
state, rows = world
|
||||||
|
_local_states(state)
|
||||||
|
db.commit()
|
||||||
|
# The server said 5, the walk got 3: m1 may simply not have been reached.
|
||||||
|
plex = _FakePlex(
|
||||||
|
[
|
||||||
|
{"ratingKey": "m3"},
|
||||||
|
{"ratingKey": "m4", "viewCount": 1, "lastViewedAt": 1_700_000_000},
|
||||||
|
{"ratingKey": "m5", "viewOffset": 120_000, "lastViewedAt": 1_700_000_000},
|
||||||
|
],
|
||||||
|
total=5,
|
||||||
|
)
|
||||||
|
out = _run(db, monkeypatch, plex)
|
||||||
|
after = rows()
|
||||||
|
assert after["m1"] == ("watched", 0, True) # NOT un-watched on a partial picture
|
||||||
|
assert after["m4"] == ("watched", 0, True) and after["m5"] == ("new", 120, True) # pulls stand
|
||||||
|
assert after["m2"] == ("watched", 0, True) # the dirty row is still pushed up
|
||||||
|
assert out["cleared"] == 0 and out["incomplete"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
_LISTED = [
|
||||||
|
{"ratingKey": "m3"},
|
||||||
|
{"ratingKey": "m4", "viewCount": 1, "lastViewedAt": 1_700_000_000},
|
||||||
|
{"ratingKey": "m5", "viewOffset": 120_000, "lastViewedAt": 1_700_000_000},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_genuinely_empty_second_section_still_lets_the_reconcile_clear(db, monkeypatch, world):
|
||||||
|
# The server SAYS the shows section holds 0: that is a complete listing, not a hiccup — before
|
||||||
|
# the E32 review it read as "incomplete" forever and un-watches never synced again.
|
||||||
|
state, rows = world
|
||||||
|
_local_states(state)
|
||||||
|
db.commit()
|
||||||
|
out = _run(db, monkeypatch, _FakePlex(_LISTED, more=[("show", [], 0)]))
|
||||||
|
assert "m1" not in rows()
|
||||||
|
assert out["cleared"] == 1 and "incomplete" not in out
|
||||||
|
|
||||||
|
|
||||||
|
def _watched_episode_in_a_shows_section(db) -> int:
|
||||||
|
"""A second mirrored section (key "2") holding one episode the owner has watched, synced."""
|
||||||
|
shows = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||||
|
db.add(shows)
|
||||||
|
db.flush()
|
||||||
|
ep = PlexItem(rating_key="e1", library_id=shows.id, kind="episode", title="e1")
|
||||||
|
db.add(ep)
|
||||||
|
db.flush()
|
||||||
|
user_id = db.execute(select(PlexLink.user_id)).scalar_one()
|
||||||
|
db.add(PlexState(user_id=user_id, item_id=ep.id, status="watched", synced_to_plex=True))
|
||||||
|
return ep.id
|
||||||
|
|
||||||
|
|
||||||
|
def _state_of(db, item_id: int) -> str | None:
|
||||||
|
db.expire_all()
|
||||||
|
st = db.execute(select(PlexState).where(PlexState.item_id == item_id)).scalar_one_or_none()
|
||||||
|
return st.status if st else None
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_section_we_mirror_that_says_0_keeps_its_rows_and_does_not_hold_up_the_rest(
|
||||||
|
db, monkeypatch, world
|
||||||
|
):
|
||||||
|
# A total of 0 from a section our items come from is a Plex that lost the library for a moment,
|
||||||
|
# not one that un-watched all of it (E32 review round 2): its rows stay. And it is decided per
|
||||||
|
# section: before round 3 one such section (a library emptied on Plex, its mirrored items kept)
|
||||||
|
# made EVERY run incomplete, and un-watches in every other section never synced again.
|
||||||
|
state, rows = world
|
||||||
|
_local_states(state)
|
||||||
|
ep_id = _watched_episode_in_a_shows_section(db)
|
||||||
|
db.commit()
|
||||||
|
out = _run(db, monkeypatch, _FakePlex(_LISTED, more=[("show", [], 0)]))
|
||||||
|
assert _state_of(db, ep_id) == "watched"
|
||||||
|
assert "m1" not in rows() # the movies section was walked whole: Plex un-watched m1
|
||||||
|
assert out["cleared"] == 1 and out["incomplete"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_library_the_allowlist_leaves_out_keeps_its_watch_states(db, monkeypatch, world):
|
||||||
|
# Its items are absent from every scan because the scan never walks it — not because Plex
|
||||||
|
# un-watched them. It cleared them on every run (E32 review round 2).
|
||||||
|
state, _rows = world
|
||||||
|
other = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||||
|
db.add(other)
|
||||||
|
db.flush()
|
||||||
|
ep = PlexItem(rating_key="e1", library_id=other.id, kind="episode", title="e1")
|
||||||
|
db.add(ep)
|
||||||
|
db.flush()
|
||||||
|
user_id = db.execute(select(PlexLink.user_id)).scalar_one()
|
||||||
|
db.add(PlexState(user_id=user_id, item_id=ep.id, status="watched", synced_to_plex=True))
|
||||||
|
_local_states(state)
|
||||||
|
sysconfig.set_value(db, "plex_libraries", "1")
|
||||||
|
db.commit()
|
||||||
|
out = _run(db, monkeypatch, _FakePlex(_LISTED, more=[("show", [], 0)]))
|
||||||
|
db.expire_all()
|
||||||
|
left = {
|
||||||
|
rk
|
||||||
|
for rk in db.execute(
|
||||||
|
select(PlexItem.rating_key).join(PlexState, PlexState.item_id == PlexItem.id)
|
||||||
|
).scalars()
|
||||||
|
}
|
||||||
|
assert "m1" not in left # the walked section still clears
|
||||||
|
kept = db.execute(select(PlexState).where(PlexState.item_id == ep.id)).scalar_one_or_none()
|
||||||
|
assert kept is not None and kept.status == "watched"
|
||||||
|
assert out["cleared"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_cut_short_section_is_not_rescued_by_a_complete_one_after_it(db, monkeypatch, world):
|
||||||
|
# Each section settles on its own walk: movies cut short (5 said, 3 handed), then a complete
|
||||||
|
# shows listing — m1 may simply not have been reached.
|
||||||
|
state, rows = world
|
||||||
|
_local_states(state)
|
||||||
|
db.commit()
|
||||||
|
shows = [{"ratingKey": "s1"}]
|
||||||
|
out = _run(db, monkeypatch, _FakePlex(_LISTED, total=5, more=[("show", shows, 1)]))
|
||||||
|
assert rows()["m1"] == ("watched", 0, True)
|
||||||
|
assert out["cleared"] == 0 and out["incomplete"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_listing_without_a_total_is_never_complete(db, monkeypatch, world):
|
||||||
|
state, rows = world
|
||||||
|
_local_states(state)
|
||||||
|
ep_id = _watched_episode_in_a_shows_section(db)
|
||||||
|
db.commit()
|
||||||
|
out = _run(db, monkeypatch, _FakePlex(_LISTED, more=[("show", [], None)]))
|
||||||
|
assert _state_of(db, ep_id) == "watched" # its section never said how much it holds
|
||||||
|
assert "m1" not in rows()
|
||||||
|
assert out["cleared"] == 1 and out["incomplete"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_empty_listing_is_not_a_library_of_unwatched_titles(db, monkeypatch, world):
|
||||||
|
state, rows = world
|
||||||
|
state("m1", status="watched", synced_to_plex=True)
|
||||||
|
db.commit()
|
||||||
|
out = _run(db, monkeypatch, _FakePlex([]))
|
||||||
|
assert rows() == {"m1": ("watched", 0, True)}
|
||||||
|
assert out["cleared"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_dirty_resume_is_re_pushed_as_a_timeline_and_a_dirty_reset_as_an_unscrobble(
|
||||||
|
db, monkeypatch, world
|
||||||
|
):
|
||||||
|
state, rows = world
|
||||||
|
state("m1", status="new", position_seconds=300, synced_to_plex=False)
|
||||||
|
state("m2", status="new", position_seconds=2, synced_to_plex=False) # under the 5 s floor
|
||||||
|
db.commit()
|
||||||
|
plex = _FakePlex([{"ratingKey": rk} for rk in ("m1", "m2", "m3", "m4", "m5")])
|
||||||
|
_run(db, monkeypatch, plex)
|
||||||
|
assert ("timeline", "m1", 300_000) in plex.calls
|
||||||
|
assert ("unscrobble", "m2") in plex.calls
|
||||||
|
assert {rk: synced for rk, (_s, _p, synced) in rows().items()} == {"m1": True, "m2": True}
|
||||||
|
|
||||||
|
|
||||||
|
def test_watch_fields_map_to_a_state_only_when_plex_holds_a_signal():
|
||||||
|
to = watch_sync._plex_watch_to_state
|
||||||
|
assert to(None, None, None) is None
|
||||||
|
assert to(0, 4_000, None) is None # a stray few seconds is not "in progress"
|
||||||
|
assert to(0, 90_000, 1_700_000_000)[:2] == ("new", 90)
|
||||||
|
assert to("2", 90_000, 1_700_000_000)[:2] == ("watched", 0) # watched wins over an offset
|
||||||
@@ -51,7 +51,12 @@ def test_a_link_whose_account_was_deleted_is_reported_dead(client, db, make_user
|
|||||||
|
|
||||||
def test_a_spent_link_is_reported_dead(client, db, make_user):
|
def test_a_spent_link_is_reported_dead(client, db, make_user):
|
||||||
raw = _issue(db, make_user(email="spent@example.com", password=_PW))
|
raw = _issue(db, make_user(email="spent@example.com", password=_PW))
|
||||||
client.post("/auth/password-reset/confirm", json={"token": raw, "password": _NEW})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/auth/password-reset/confirm", json={"token": raw, "password": _NEW}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
|
||||||
assert client.post("/auth/password-reset/check", json={"token": raw}).json() == {"ok": False}
|
assert client.post("/auth/password-reset/check", json={"token": raw}).json() == {"ok": False}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
"""No test sends a request and throws the response away (E32 S2).
|
||||||
|
|
||||||
|
A discarded `client.post(...)` passes whatever it got back. As a SETUP step, a failure leaves the
|
||||||
|
test asserting on a state it never reached; as the step UNDER test, the test proves nothing. E32
|
||||||
|
found `test_youtube_sources_never_consult_the_guard` passing on a 400 that stopped the request
|
||||||
|
before the code it meant to exercise, and 53 more discarded responses across the suite. Each one
|
||||||
|
now states the status it expects; this keeps it that way.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import ast
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
TESTS = Path(__file__).resolve().parent
|
||||||
|
VERBS = {"get", "post", "put", "patch", "delete", "head", "options", "request"}
|
||||||
|
|
||||||
|
|
||||||
|
def discarded_requests(source: str) -> list[int]:
|
||||||
|
"""Line numbers of the statements that are a bare request call on an app path."""
|
||||||
|
hits = []
|
||||||
|
for node in ast.walk(ast.parse(source)):
|
||||||
|
if not (isinstance(node, ast.Expr) and isinstance(node.value, ast.Call)):
|
||||||
|
continue
|
||||||
|
func = node.value.func
|
||||||
|
if not (isinstance(func, ast.Attribute) and func.attr in VERBS):
|
||||||
|
continue
|
||||||
|
path = node.value.args[0] if node.value.args else None
|
||||||
|
literal = path.value if isinstance(path, ast.Constant) else None
|
||||||
|
if isinstance(path, ast.JoinedStr):
|
||||||
|
first = path.values[0] if path.values else None
|
||||||
|
literal = first.value if isinstance(first, ast.Constant) else None
|
||||||
|
if isinstance(literal, str) and literal.startswith(("/api", "/auth")):
|
||||||
|
hits.append(node.lineno)
|
||||||
|
return hits
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_detector_sees_a_discarded_request_and_passes_a_checked_one():
|
||||||
|
src = (
|
||||||
|
"client.post('/api/x')\n"
|
||||||
|
"client.get(f'/auth/{u}')\n"
|
||||||
|
"assert client.post('/api/x').status_code == 200\n"
|
||||||
|
"r = client.get('/api/y')\n"
|
||||||
|
"items.get('/api/not-a-request-but-a-key')\n"
|
||||||
|
)
|
||||||
|
# The last line is a false positive by design: a dict keyed by an app path is not a thing the
|
||||||
|
# suite does, and a narrower match would miss a client held under any other name.
|
||||||
|
assert discarded_requests(src) == [1, 2, 5]
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_test_discards_a_response():
|
||||||
|
offenders = [
|
||||||
|
f"{p.name}:{line}"
|
||||||
|
for p in sorted(TESTS.rglob("test_*.py"))
|
||||||
|
for line in discarded_requests(p.read_text(encoding="utf-8"))
|
||||||
|
]
|
||||||
|
assert offenders == [], (
|
||||||
|
"assert the status these requests expect (`assert client.post(...).status_code == 200`), "
|
||||||
|
f"or keep the response and check it: {offenders}"
|
||||||
|
)
|
||||||
@@ -100,15 +100,22 @@ def test_a_later_registration_never_touches_an_existing_password(db, client, smt
|
|||||||
have installed a password where the whole point is that there is none. Reverted; the owner's
|
have installed a password where the whole point is that there is none. Reverted; the owner's
|
||||||
recourse is password-reset, which proves the mailbox. Residual logged as SEC-16.
|
recourse is password-reset, which proves the mailbox. Residual logged as SEC-16.
|
||||||
"""
|
"""
|
||||||
client.post(
|
assert (
|
||||||
"/auth/register", json={"email": "victim@example.com", "password": "the first password"}
|
client.post(
|
||||||
|
"/auth/register", json={"email": "victim@example.com", "password": "the first password"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
user = db.execute(select(User).where(User.email == "victim@example.com")).scalar_one()
|
user = db.execute(select(User).where(User.email == "victim@example.com")).scalar_one()
|
||||||
before = user.password_hash
|
before = user.password_hash
|
||||||
|
|
||||||
client.post(
|
assert (
|
||||||
"/auth/register", json={"email": "victim@example.com", "password": "the second password"}
|
client.post(
|
||||||
|
"/auth/register",
|
||||||
|
json={"email": "victim@example.com", "password": "the second password"},
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
|
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
@@ -116,8 +123,11 @@ def test_a_later_registration_never_touches_an_existing_password(db, client, smt
|
|||||||
|
|
||||||
|
|
||||||
def test_a_later_registration_leaves_a_link_already_in_flight_alive(db, client, smtp_on, wire):
|
def test_a_later_registration_leaves_a_link_already_in_flight_alive(db, client, smtp_on, wire):
|
||||||
client.post(
|
assert (
|
||||||
"/auth/register", json={"email": "victim@example.com", "password": "the first password"}
|
client.post(
|
||||||
|
"/auth/register", json={"email": "victim@example.com", "password": "the first password"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
user = db.execute(select(User).where(User.email == "victim@example.com")).scalar_one()
|
user = db.execute(select(User).where(User.email == "victim@example.com")).scalar_one()
|
||||||
@@ -126,8 +136,12 @@ def test_a_later_registration_leaves_a_link_already_in_flight_alive(db, client,
|
|||||||
# would go green while asserting nothing the day registration stops issuing a token.
|
# would go green while asserting nothing the day registration stops issuing a token.
|
||||||
assert len(first) == 1
|
assert len(first) == 1
|
||||||
|
|
||||||
client.post(
|
assert (
|
||||||
"/auth/register", json={"email": "victim@example.com", "password": "the second password"}
|
client.post(
|
||||||
|
"/auth/register",
|
||||||
|
json={"email": "victim@example.com", "password": "the second password"},
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
|
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
@@ -148,8 +162,12 @@ def test_a_stripped_account_keeps_its_recovery_stamp(db, client, smtp_on, make_u
|
|||||||
password_revoked_at=datetime.now(timezone.utc),
|
password_revoked_at=datetime.now(timezone.utc),
|
||||||
)
|
)
|
||||||
|
|
||||||
client.post(
|
assert (
|
||||||
"/auth/register", json={"email": "stripped@example.com", "password": "an attacker password"}
|
client.post(
|
||||||
|
"/auth/register",
|
||||||
|
json={"email": "stripped@example.com", "password": "an attacker password"},
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
|
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
@@ -169,7 +187,9 @@ def test_a_denied_sibling_is_not_reopened_while_a_pending_row_exists(db, client)
|
|||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
|
||||||
client.post("/auth/request-access", json={"email": "pairone@gmail.com"})
|
assert (
|
||||||
|
client.post("/auth/request-access", json={"email": "pairone@gmail.com"}).status_code == 200
|
||||||
|
)
|
||||||
|
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
rows = {i.email: i.status for i in db.execute(select(Invite)).scalars().all()}
|
rows = {i.email: i.status for i in db.execute(select(Invite)).scalars().all()}
|
||||||
@@ -195,7 +215,7 @@ def test_an_env_granted_address_cannot_be_shadowed_by_a_stranger(db, client, mon
|
|||||||
# env grant. With the env list matched canonically the request never opens a row.
|
# env grant. With the env list matched canonically the request never opens a row.
|
||||||
monkeypatch.setattr(settings, "allowed_emails", "bob@gmail.com")
|
monkeypatch.setattr(settings, "allowed_emails", "bob@gmail.com")
|
||||||
|
|
||||||
client.post("/auth/request-access", json={"email": "b.ob@gmail.com"})
|
assert client.post("/auth/request-access", json={"email": "b.ob@gmail.com"}).status_code == 200
|
||||||
|
|
||||||
assert db.execute(select(Invite)).scalars().all() == []
|
assert db.execute(select(Invite)).scalars().all() == []
|
||||||
assert auth_core.is_allowed(db, "bob@gmail.com") is True
|
assert auth_core.is_allowed(db, "bob@gmail.com") is True
|
||||||
@@ -331,7 +351,12 @@ def test_two_admins_two_languages_two_mails(db, client, make_user, smtp_on, wire
|
|||||||
make_user(email="hu.admin@example.com", role="admin", preferences={"language": "hu"})
|
make_user(email="hu.admin@example.com", role="admin", preferences={"language": "hu"})
|
||||||
make_user(email="en.admin@example.com", role="admin", preferences={"language": "en"})
|
make_user(email="en.admin@example.com", role="admin", preferences={"language": "en"})
|
||||||
|
|
||||||
client.post("/auth/request-access", json={"email": "asking@example.com", "lang": "en"})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/auth/request-access", json={"email": "asking@example.com", "lang": "en"}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
|
||||||
notices = [w for w in wire if "asking@example.com" in w[1]]
|
notices = [w for w in wire if "asking@example.com" in w[1]]
|
||||||
assert len(notices) == 2
|
assert len(notices) == 2
|
||||||
|
|||||||
@@ -53,8 +53,8 @@ def test_default_is_per_source(db, monkeypatch, override_dep):
|
|||||||
pid = c.post("/api/saved-views", json={"name": "P", "filters": {}, "source": "plex"}).json()[
|
pid = c.post("/api/saved-views", json={"name": "P", "filters": {}, "source": "plex"}).json()[
|
||||||
"id"
|
"id"
|
||||||
]
|
]
|
||||||
c.patch(f"/api/saved-views/{fid}", json={"is_default": True})
|
assert c.patch(f"/api/saved-views/{fid}", json={"is_default": True}).status_code == 200
|
||||||
c.patch(f"/api/saved-views/{pid}", json={"is_default": True})
|
assert c.patch(f"/api/saved-views/{pid}", json={"is_default": True}).status_code == 200
|
||||||
# Making the Plex view default must NOT clear the feed view's default (different source).
|
# Making the Plex view default must NOT clear the feed view's default (different source).
|
||||||
assert c.get("/api/saved-views?source=feed").json()[0]["is_default"] is True
|
assert c.get("/api/saved-views?source=feed").json()[0]["is_default"] is True
|
||||||
assert c.get("/api/saved-views?source=plex").json()[0]["is_default"] is True
|
assert c.get("/api/saved-views?source=plex").json()[0]["is_default"] is True
|
||||||
@@ -67,3 +67,18 @@ def test_duplicate_name_within_a_source_conflicts(db, monkeypatch, override_dep)
|
|||||||
b = c.post("/api/saved-views", json={"name": "Dup", "filters": {}, "source": "plex"})
|
b = c.post("/api/saved-views", json={"name": "Dup", "filters": {}, "source": "plex"})
|
||||||
assert a.status_code == 200
|
assert a.status_code == 200
|
||||||
assert b.status_code == 409 # same (user, source, name)
|
assert b.status_code == 409 # same (user, source, name)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_feed_views_tag_pick_must_be_a_short_list_of_ids(db, monkeypatch, override_dep):
|
||||||
|
"""The autotag job reads every view's tag pick (E32 review): the route holds it to the shape the
|
||||||
|
feed sends, on create and on update."""
|
||||||
|
user = _user(db)
|
||||||
|
c = _client(db, user, monkeypatch, override_dep)
|
||||||
|
for bad in (5, "1", [1, "2"], [True], list(range(501))):
|
||||||
|
r = c.post("/api/saved-views", json={"name": "Bad", "filters": {"tags": bad}})
|
||||||
|
assert r.status_code == 422, (bad, r.text)
|
||||||
|
ok = c.post("/api/saved-views", json={"name": "Ok", "filters": {"tags": [1, 2]}})
|
||||||
|
assert ok.status_code == 200, ok.text
|
||||||
|
vid = ok.json()["id"]
|
||||||
|
assert c.patch(f"/api/saved-views/{vid}", json={"filters": {"tags": 5}}).status_code == 422
|
||||||
|
assert c.patch(f"/api/saved-views/{vid}", json={"filters": {"tags": []}}).status_code == 200
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
"""youtube/search_scrape — the zero-quota InnerTube search (E32 S2: the module was 20% covered).
|
||||||
|
|
||||||
|
Everything here breaks silently when YouTube moves: a parser that stops finding videos, a cursor that
|
||||||
|
stops paging, a fallback key that outlives its usefulness. The network is an httpx MockTransport.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.youtube import search_scrape
|
||||||
|
|
||||||
|
RESULTS_HTML = (
|
||||||
|
'<script>{"INNERTUBE_API_KEY":"live-key","INNERTUBE_CLIENT_VERSION":"2.20261001"}</script>'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _video(vid: str, title: str, *, simple: bool = False) -> dict:
|
||||||
|
return {
|
||||||
|
"videoRenderer": {
|
||||||
|
"videoId": vid,
|
||||||
|
"title": {"simpleText": title}
|
||||||
|
if simple
|
||||||
|
else {"runs": [{"text": title[:3]}, {"text": title[3:]}]},
|
||||||
|
"ownerText": {
|
||||||
|
"runs": [
|
||||||
|
{
|
||||||
|
"text": "Chan",
|
||||||
|
"navigationEndpoint": {"browseEndpoint": {"browseId": "UCchan"}},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"thumbnail": {"thumbnails": [{"url": "small.jpg"}, {"url": "big.jpg"}]},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
PAGE = {
|
||||||
|
"contents": {
|
||||||
|
"sectionListRenderer": {
|
||||||
|
"contents": [
|
||||||
|
{
|
||||||
|
"itemSectionRenderer": {
|
||||||
|
"contents": [
|
||||||
|
_video("v1", "First video"),
|
||||||
|
{"channelRenderer": {"channelId": "UCx"}}, # not a video: skipped
|
||||||
|
{
|
||||||
|
"reelShelfRenderer": {
|
||||||
|
"items": [{"reelItemRenderer": {"videoId": "s1"}}]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
_video("v2", "Second", simple=True),
|
||||||
|
_video("v1", "First video"), # a repeat in another shelf: once
|
||||||
|
{"videoRenderer": {"title": {"simpleText": "no id"}}},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"continuationItemRenderer": {
|
||||||
|
"continuationEndpoint": {"continuationCommand": {"token": "NEXT"}}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def net(monkeypatch):
|
||||||
|
"""Route every httpx.Client the module opens through a handler the test sets."""
|
||||||
|
calls: list[httpx.Request] = []
|
||||||
|
state: dict = {"handler": None}
|
||||||
|
|
||||||
|
def transport(request: httpx.Request) -> httpx.Response:
|
||||||
|
calls.append(request)
|
||||||
|
return state["handler"](request)
|
||||||
|
|
||||||
|
real = httpx.Client
|
||||||
|
monkeypatch.setattr(
|
||||||
|
search_scrape.httpx,
|
||||||
|
"Client",
|
||||||
|
lambda **kw: real(transport=httpx.MockTransport(transport), **kw),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(search_scrape, "_config", None)
|
||||||
|
state["calls"] = calls
|
||||||
|
return state
|
||||||
|
|
||||||
|
|
||||||
|
def _serve(results_html=RESULTS_HTML, search=PAGE, status=200):
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
if request.url.path == "/results":
|
||||||
|
if isinstance(results_html, Exception):
|
||||||
|
raise results_html
|
||||||
|
return httpx.Response(200, text=results_html)
|
||||||
|
if isinstance(search, str):
|
||||||
|
return httpx.Response(status, text=search)
|
||||||
|
return httpx.Response(status, json=search)
|
||||||
|
|
||||||
|
return handler
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_page_flattens_its_videos_in_order_and_hands_back_the_cursor(net):
|
||||||
|
net["handler"] = _serve()
|
||||||
|
page = search_scrape.search("cats")
|
||||||
|
assert page["next_page_token"] == "NEXT"
|
||||||
|
assert [i["id"] for i in page["items"]] == ["v1", "v2"]
|
||||||
|
assert page["items"][0] == {
|
||||||
|
"id": "v1",
|
||||||
|
"channel_id": "UCchan",
|
||||||
|
"channel_title": "Chan",
|
||||||
|
"title": "First video",
|
||||||
|
"published_at": None,
|
||||||
|
"thumbnail_url": "big.jpg",
|
||||||
|
"live_broadcast": "none",
|
||||||
|
}
|
||||||
|
assert page["items"][1]["title"] == "Second"
|
||||||
|
post = net["calls"][-1]
|
||||||
|
assert post.url.params["key"] == "live-key"
|
||||||
|
body = json.loads(post.content)
|
||||||
|
assert body["query"] == "cats" and body["params"] == search_scrape._TYPE_VIDEO
|
||||||
|
assert body["context"]["client"]["clientVersion"] == "2.20261001"
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_next_page_sends_the_cursor_not_the_query_and_the_end_has_none(net):
|
||||||
|
net["handler"] = _serve(search={"onResponseReceivedCommands": [_video("v9", "Last")]})
|
||||||
|
page = search_scrape.search("cats", continuation="NEXT")
|
||||||
|
body = json.loads(net["calls"][-1].content)
|
||||||
|
assert body["continuation"] == "NEXT" and "query" not in body
|
||||||
|
assert [i["id"] for i in page["items"]] == ["v9"]
|
||||||
|
assert page["next_page_token"] is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("search", "status"),
|
||||||
|
[
|
||||||
|
("<html>consent</html>", 200),
|
||||||
|
({"error": "blocked"}, 429),
|
||||||
|
({"responseContext": {}, "alerts": ["unusual traffic"]}, 200),
|
||||||
|
([], 200),
|
||||||
|
],
|
||||||
|
ids=["not-json", "http-error", "unknown-shape", "not-an-object"],
|
||||||
|
)
|
||||||
|
def test_a_blocked_or_garbled_answer_is_an_error_not_an_empty_page(net, search, status):
|
||||||
|
net["handler"] = _serve(search=search, status=status)
|
||||||
|
with pytest.raises(search_scrape.ScrapeError):
|
||||||
|
search_scrape.search("cats")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_search_with_no_results_is_an_empty_page_not_an_error(net):
|
||||||
|
net["handler"] = _serve(search={"contents": {"sectionListRenderer": {"contents": []}}})
|
||||||
|
assert search_scrape.search("zzzzqqq") == {"items": [], "next_page_token": None}
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_live_key_is_fetched_once_and_cached(net):
|
||||||
|
net["handler"] = _serve()
|
||||||
|
search_scrape.search("a")
|
||||||
|
search_scrape.search("b")
|
||||||
|
assert [c.url.path for c in net["calls"]].count("/results") == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"results_html",
|
||||||
|
[httpx.ConnectError("down"), "<html>a consent page, no keys</html>"],
|
||||||
|
ids=["fetch-failed", "no-key-on-page"],
|
||||||
|
)
|
||||||
|
def test_a_failed_key_fetch_falls_back_for_now_and_tries_again_next_time(net, results_html):
|
||||||
|
"""The fallback pair used to be CACHED for the life of the process: one failed fetch pinned
|
||||||
|
every later search to a hard-coded key, and once that went stale search returned nothing,
|
||||||
|
with no error anywhere (E32 S2)."""
|
||||||
|
net["handler"] = _serve(results_html=results_html)
|
||||||
|
search_scrape.search("a")
|
||||||
|
assert net["calls"][-1].url.params["key"] == search_scrape._FALLBACK_KEY
|
||||||
|
|
||||||
|
net["handler"] = _serve()
|
||||||
|
search_scrape.search("b")
|
||||||
|
assert net["calls"][-1].url.params["key"] == "live-key"
|
||||||
@@ -68,8 +68,12 @@ def test_using_the_link_evicts_whoever_held_the_old_password(db, as_user, make_u
|
|||||||
admin = make_user(role="admin")
|
admin = make_user(role="admin")
|
||||||
db.commit()
|
db.commit()
|
||||||
token = _link(as_user(admin), target.id)
|
token = _link(as_user(admin), target.id)
|
||||||
client.post(
|
assert (
|
||||||
"/auth/password-reset/confirm", json={"token": token, "password": "the owner's own choice"}
|
client.post(
|
||||||
|
"/auth/password-reset/confirm",
|
||||||
|
json={"token": token, "password": "the owner's own choice"},
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
)
|
)
|
||||||
|
|
||||||
db.refresh(target)
|
db.refresh(target)
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
"""youtube/shorts.probe_is_short — three answers, and the third is not the second (E32 S2: 33% covered).
|
||||||
|
|
||||||
|
A Short answers 200 at /shorts/<id>; a regular video redirects to /watch. Anything else — an error
|
||||||
|
page, a network failure — is "undetermined, ask again", never "not a Short": that would file a Short
|
||||||
|
in the normal feed for good.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.youtube import shorts
|
||||||
|
|
||||||
|
|
||||||
|
def _client(respond) -> httpx.Client:
|
||||||
|
return httpx.Client(transport=httpx.MockTransport(respond))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("status", "expected"),
|
||||||
|
[(200, True), (301, False), (302, False), (303, False), (307, False), (308, False)]
|
||||||
|
+ [(404, None), (429, None), (500, None)],
|
||||||
|
)
|
||||||
|
def test_the_status_decides_short_regular_or_undetermined(status, expected):
|
||||||
|
seen: list[httpx.Request] = []
|
||||||
|
|
||||||
|
def respond(request: httpx.Request) -> httpx.Response:
|
||||||
|
seen.append(request)
|
||||||
|
return httpx.Response(status, headers={"location": "https://www.youtube.com/watch?v=x"})
|
||||||
|
|
||||||
|
assert shorts.probe_is_short(_client(respond), "abc123") is expected
|
||||||
|
assert str(seen[0].url) == "https://www.youtube.com/shorts/abc123"
|
||||||
|
assert len(seen) == 1 # a redirect is the answer, never followed
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_network_failure_is_undetermined_not_regular():
|
||||||
|
def respond(request: httpx.Request) -> httpx.Response:
|
||||||
|
raise httpx.ConnectTimeout("slow", request=request)
|
||||||
|
|
||||||
|
assert shorts.probe_is_short(_client(respond), "abc123") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_client_skips_the_consent_interstitial():
|
||||||
|
with shorts.make_client() as c:
|
||||||
|
assert "SOCS=" in c.headers["cookie"]
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
"""The size / complexity RATCHET for the backend (E32, owner 2026-10-03 — "class size" from the quality
|
||||||
|
checklist, measured rather than argued). The frontend's twin is src/lib/sourceSize.contract.test.ts.
|
||||||
|
|
||||||
|
Two limits, each with today's offenders as a CEILING:
|
||||||
|
* a module over MAX_LINES lines — a listed one may not grow past its line count; any other module
|
||||||
|
may not cross the limit;
|
||||||
|
* a function over ruff's C901 limit (cyclomatic complexity, MAX_COMPLEXITY) — a listed one, by its
|
||||||
|
NAME (`module::function`), may not rise; no other function may cross the limit. (Ranked per
|
||||||
|
module until the E32 review: one function could grow by what another in the same module lost.)
|
||||||
|
A rename reads as a new offender plus a stale entry: move its row.
|
||||||
|
Shrinking is free; a module or function that drops UNDER the limit must be taken off its list (red
|
||||||
|
until it is), so the lists only get shorter. Raising a ceiling is a decision for the owner, with the
|
||||||
|
reason in docs/gates.md.
|
||||||
|
|
||||||
|
C901 is measured here rather than switched on in ruff.toml because ruff has no per-function
|
||||||
|
baseline: on, it would need a `noqa` on each of today's offenders, and a `noqa` lets the function
|
||||||
|
grow without limit. This keeps the ceiling on the number.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
BACKEND = Path(__file__).resolve().parent.parent
|
||||||
|
MAX_LINES = 800
|
||||||
|
MAX_COMPLEXITY = 10 # ruff's C901 default (mccabe max-complexity)
|
||||||
|
|
||||||
|
BIG_MODULES: dict[str, int] = {
|
||||||
|
"app/downloads/edit.py": 833,
|
||||||
|
"app/downloads/pipeline.py": 870,
|
||||||
|
"app/logs.py": 839,
|
||||||
|
"app/models.py": 1607,
|
||||||
|
"app/plex/stream.py": 1128,
|
||||||
|
"app/plex/subtitles.py": 1092,
|
||||||
|
"app/routes/admin.py": 864,
|
||||||
|
"app/routes/auth.py": 1353,
|
||||||
|
"app/routes/downloads.py": 1237,
|
||||||
|
}
|
||||||
|
|
||||||
|
COMPLEX: dict[str, int] = {
|
||||||
|
"app/bodylimit.py::__call__": 11,
|
||||||
|
"app/discover/hero.py::pick": 14,
|
||||||
|
"app/discover/pins.py::_shelf_card": 12,
|
||||||
|
"app/downloads/edit.py::normalize_edit_spec": 16,
|
||||||
|
"app/downloads/edit.py::run_ffmpeg": 15,
|
||||||
|
"app/downloads/formats.py::build_ydl_opts": 11,
|
||||||
|
"app/downloads/gc.py::_sweep_orphan_files": 15,
|
||||||
|
"app/downloads/gc.py::run_download_gc": 11,
|
||||||
|
"app/downloads/og.py::_jpeg_size": 12,
|
||||||
|
"app/downloads/og.py::_watch_tags": 11,
|
||||||
|
"app/downloads/pipeline.py::_download": 14,
|
||||||
|
"app/downloads/pipeline.py::_process_edit": 12,
|
||||||
|
"app/feedquery.py::feed_summary": 14,
|
||||||
|
"app/feedquery.py::filtered_query": 28,
|
||||||
|
"app/plex/query.py::meta_filter_conds": 13,
|
||||||
|
"app/plex/stream.py::start_session": 14,
|
||||||
|
"app/plex/subtitles.py::_ffmpeg_extract": 12,
|
||||||
|
"app/plex/sync.py::sync": 13,
|
||||||
|
"app/plex/tmdb.py::parse_tmdb": 11,
|
||||||
|
"app/plex/watch_sync.py::run_plex_watch_sync": 12,
|
||||||
|
"app/routes/auth.py::callback": 19,
|
||||||
|
"app/routes/debug_sessions.py::send_issue": 12,
|
||||||
|
"app/routes/downloads.py::resolve_source": 11,
|
||||||
|
"app/routes/plex/browse.py::facets": 25,
|
||||||
|
"app/routes/plex/browse.py::per_table": 13,
|
||||||
|
"app/routes/plex/browse.py::unified_library": 18,
|
||||||
|
"app/routes/plex/detail.py::item_detail": 13,
|
||||||
|
"app/routes/search.py::search_youtube": 15,
|
||||||
|
"app/sync/playlists.py::merge_playlist": 19,
|
||||||
|
"app/sync/runner.py::run_rss_poll": 13,
|
||||||
|
"app/sync/tagsuggest.py::compute_tag_suggestions": 14,
|
||||||
|
"app/worker.py::_run_supervised": 18,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _line_counts() -> dict[str, int]:
|
||||||
|
return {
|
||||||
|
p.relative_to(BACKEND).as_posix(): len(p.read_text(encoding="utf-8").split("\n"))
|
||||||
|
for p in sorted((BACKEND / "app").rglob("*.py"))
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _complexity() -> dict[str, int]:
|
||||||
|
"""`module::function` -> its complexity; a name taken twice in a module gets ` 2`, ` 3` in
|
||||||
|
source order."""
|
||||||
|
out = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
"-m",
|
||||||
|
"ruff",
|
||||||
|
"check",
|
||||||
|
"--config",
|
||||||
|
str(BACKEND / "ruff.toml"),
|
||||||
|
"--select",
|
||||||
|
"C901",
|
||||||
|
"--output-format",
|
||||||
|
"json",
|
||||||
|
"--exit-zero",
|
||||||
|
str(BACKEND / "app"),
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=True,
|
||||||
|
).stdout
|
||||||
|
found: dict[str, int] = {}
|
||||||
|
for m in sorted(json.loads(out), key=lambda m: (m["filename"], m["location"]["row"])):
|
||||||
|
value = re.search(r"\((\d+) > \d+\)", m["message"])
|
||||||
|
name = re.search(r"`([^`]+)`", m["message"])
|
||||||
|
assert value and name, m["message"]
|
||||||
|
rel = Path(m["filename"]).resolve().relative_to(BACKEND).as_posix()
|
||||||
|
key, n = f"{rel}::{name.group(1)}", 2
|
||||||
|
while key in found:
|
||||||
|
key, n = f"{rel}::{name.group(1)} {n}", n + 1
|
||||||
|
found[key] = int(value.group(1))
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_module_crosses_the_line_limit_and_a_listed_one_does_not_grow():
|
||||||
|
over = [
|
||||||
|
f"{f}: {n} lines (ceiling {BIG_MODULES.get(f, MAX_LINES)})"
|
||||||
|
for f, n in _line_counts().items()
|
||||||
|
if n > BIG_MODULES.get(f, MAX_LINES)
|
||||||
|
]
|
||||||
|
assert over == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_lists_no_module_that_is_back_under_the_limit():
|
||||||
|
lines = _line_counts()
|
||||||
|
assert [f for f in BIG_MODULES if lines.get(f, 0) <= MAX_LINES] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_function_crosses_the_complexity_limit_and_a_listed_one_does_not_rise():
|
||||||
|
worse = [
|
||||||
|
f"{fn}: {v} (ceiling {COMPLEX.get(fn, MAX_COMPLEXITY)})"
|
||||||
|
for fn, v in _complexity().items()
|
||||||
|
if v > COMPLEX.get(fn, MAX_COMPLEXITY)
|
||||||
|
]
|
||||||
|
assert worse == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_lists_no_function_that_is_back_under_the_limit_or_gone():
|
||||||
|
actual = _complexity()
|
||||||
|
assert [fn for fn in COMPLEX if fn not in actual] == []
|
||||||
@@ -229,7 +229,12 @@ def test_registering_again_does_not_create_a_second_account(db, client, monkeypa
|
|||||||
_smtp_on(monkeypatch)
|
_smtp_on(monkeypatch)
|
||||||
_capture(monkeypatch, "send_verify_email")
|
_capture(monkeypatch, "send_verify_email")
|
||||||
for _ in range(2):
|
for _ in range(2):
|
||||||
client.post("/auth/register", json={"email": "twice@example.com", "password": _PW})
|
assert (
|
||||||
|
client.post(
|
||||||
|
"/auth/register", json={"email": "twice@example.com", "password": _PW}
|
||||||
|
).status_code
|
||||||
|
== 200
|
||||||
|
)
|
||||||
|
|
||||||
db.expire_all()
|
db.expire_all()
|
||||||
rows = db.execute(select(User).where(User.email == "twice@example.com")).scalars().all()
|
rows = db.execute(select(User).where(User.email == "twice@example.com")).scalars().all()
|
||||||
|
|||||||
@@ -8,7 +8,9 @@ import httpx
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.auth import has_read_scope
|
from app.auth import has_read_scope
|
||||||
from app.models import OAuthToken
|
from app.models import Channel, OAuthToken
|
||||||
|
from app.sync.explore import explore_ingest_page
|
||||||
|
from app.sync.videos import backfill_channel_deep, backfill_channel_recent
|
||||||
from app.youtube import client as yt
|
from app.youtube import client as yt
|
||||||
from app.youtube.client import YouTubeError, refusal_reason
|
from app.youtube.client import YouTubeError, refusal_reason
|
||||||
|
|
||||||
@@ -27,12 +29,91 @@ from app.youtube.client import YouTubeError, refusal_reason
|
|||||||
(YouTubeError("No refresh token; user must re-authenticate"), "reconnect"),
|
(YouTubeError("No refresh token; user must re-authenticate"), "reconnect"),
|
||||||
(YouTubeError("m", reason="transport"), "unavailable"),
|
(YouTubeError("m", reason="transport"), "unavailable"),
|
||||||
(YouTubeError("m", reason="backendError", path="subscriptions"), "unavailable"),
|
(YouTubeError("m", reason="backendError", path="subscriptions"), "unavailable"),
|
||||||
|
# A missing playlist the user never named (a channel's uploads): nothing they can act on.
|
||||||
|
(YouTubeError("m", reason="playlistNotFound", path="playlistItems"), "unavailable"),
|
||||||
|
(YouTubeError("m", reason="emptyAnswer", path="playlistItems"), "unavailable"),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_a_failure_is_sorted_by_what_the_user_can_do(exc, reason):
|
def test_a_failure_is_sorted_by_what_the_user_can_do(exc, reason):
|
||||||
assert refusal_reason(exc) == reason
|
assert refusal_reason(exc) == reason
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_missing_playlist_is_gone_only_on_a_route_about_that_playlist():
|
||||||
|
# A playlist deleted on YouTube: neither a retry nor a reconnect helps (it said "reconnect").
|
||||||
|
# Said only where the user picked that one list — anywhere else "this playlist no longer
|
||||||
|
# exists" names nothing on their screen (E32 review round 2).
|
||||||
|
exc = YouTubeError("m", reason="playlistNotFound", path="playlists")
|
||||||
|
assert refusal_reason(exc, playlist=True) == "gone"
|
||||||
|
assert refusal_reason(exc) == "unavailable"
|
||||||
|
assert refusal_reason(YouTubeError("m", reason="quotaExceeded"), playlist=True) == "quota"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channel_without_uploads_explores_as_an_empty_page():
|
||||||
|
# YouTube answers a channel's uploads playlist as not found when the channel has no uploads.
|
||||||
|
class _Yt:
|
||||||
|
def get_playlist_items(self, *_):
|
||||||
|
raise YouTubeError("m", reason="playlistNotFound", path="playlistItems")
|
||||||
|
|
||||||
|
channel = SimpleNamespace(id="UCempty", uploads_playlist_id="UUempty")
|
||||||
|
assert explore_ingest_page(None, _Yt(), channel) == {"ingested": 0, "next_page_token": None}
|
||||||
|
|
||||||
|
|
||||||
|
class _NoUploads:
|
||||||
|
"""A client whose every uploads read answers "playlist not found" (a channel with no uploads)."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.calls = 0
|
||||||
|
|
||||||
|
def get_playlist_items(self, *_):
|
||||||
|
self.calls += 1
|
||||||
|
raise YouTubeError("m", reason="playlistNotFound", path="playlistItems")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channel_without_uploads_backfills_as_done_and_is_not_charged_again(db):
|
||||||
|
# The recent pass stamped nothing on this answer, so the channel was picked — and charged a
|
||||||
|
# quota unit, with an ERROR traceback — on every backfill run, forever (E32 review round 3).
|
||||||
|
ch = Channel(id="UCempty", title="Empty", uploads_playlist_id="UUempty")
|
||||||
|
db.add(ch)
|
||||||
|
db.commit()
|
||||||
|
yt = _NoUploads()
|
||||||
|
assert backfill_channel_recent(db, yt, ch) == 0
|
||||||
|
db.refresh(ch)
|
||||||
|
assert ch.recent_synced_at is not None and ch.backfill_done and ch.backfill_cursor is None
|
||||||
|
assert backfill_channel_deep(db, yt, ch) == 0
|
||||||
|
assert yt.calls == 1 # the deep pass has nothing left to ask
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_deep_backfill_whose_uploads_vanish_ends_instead_of_failing(db):
|
||||||
|
ch = Channel(id="UCgone", title="Gone", uploads_playlist_id="UUgone", backfill_cursor="page2")
|
||||||
|
db.add(ch)
|
||||||
|
db.commit()
|
||||||
|
assert backfill_channel_deep(db, _NoUploads(), ch) == 0
|
||||||
|
db.refresh(ch)
|
||||||
|
assert ch.backfill_done and ch.backfill_cursor is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_any_other_backfill_failure_still_raises(db):
|
||||||
|
class _Yt:
|
||||||
|
def get_playlist_items(self, *_):
|
||||||
|
raise YouTubeError("m", reason="quotaExceeded", path="playlistItems")
|
||||||
|
|
||||||
|
ch = Channel(id="UCq", title="Q", uploads_playlist_id="UUq")
|
||||||
|
db.add(ch)
|
||||||
|
db.commit()
|
||||||
|
with pytest.raises(YouTubeError):
|
||||||
|
backfill_channel_recent(db, _Yt(), ch)
|
||||||
|
|
||||||
|
|
||||||
|
def test_any_other_explore_failure_still_raises():
|
||||||
|
class _Yt:
|
||||||
|
def get_playlist_items(self, *_):
|
||||||
|
raise YouTubeError("m", reason="quotaExceeded", path="playlistItems")
|
||||||
|
|
||||||
|
channel = SimpleNamespace(id="UCx", uploads_playlist_id="UUx")
|
||||||
|
with pytest.raises(YouTubeError):
|
||||||
|
explore_ingest_page(None, _Yt(), channel)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("route", "target"),
|
("route", "target"),
|
||||||
[
|
[
|
||||||
|
|||||||
+5
-2
@@ -536,8 +536,10 @@ of a full-width panel and nothing on the right.
|
|||||||
## Dialogs — _Decided_ (C4, built in S3f)
|
## Dialogs — _Decided_ (C4, built in S3f)
|
||||||
|
|
||||||
- **Initial focus by purpose** (`Modal`'s required `initialFocus`, so no dialog inherits "the first
|
- **Initial focus by purpose** (`Modal`'s required `initialFocus`, so no dialog inherits "the first
|
||||||
focusable", which is always the ✕): `field` — a form opens on the body's first control, and one that
|
focusable", which is always the ✕): `field` — a form opens on its first field: the body's first
|
||||||
loads after opening moves the focus there when it arrives, as long as the focus still rests on the ✕;
|
control, or the one marked `data-initial-focus` when a control above the form is not one of its
|
||||||
|
fields (the issue composer's Edit / Preview switch); one that loads after opening moves the focus
|
||||||
|
there when it arrives, as long as the focus still rests on the ✕;
|
||||||
`action` — a confirm, where the least destructive button takes `autoFocus`; `close` — anything to read,
|
`action` — a confirm, where the least destructive button takes `autoFocus`; `close` — anything to read,
|
||||||
browse or manage. [WAI-ARIA APG]
|
browse or manage. [WAI-ARIA APG]
|
||||||
- **The header and the footer are fixed; only the body scrolls,** with the B9 fade at each edge, so the
|
- **The header and the footer are fixed; only the body scrolls,** with the B9 fade at each edge, so the
|
||||||
@@ -702,3 +704,4 @@ of a full-width panel and nothing on the right.
|
|||||||
| 2026-10-01 | E31 Nav rail rows are links | Each page row is a real link: a middle-click or Ctrl-click opens the page in a new tab | A button | The owner: open a page in a tab of its own, as on any site |
|
| 2026-10-01 | E31 Nav rail rows are links | Each page row is a real link: a middle-click or Ctrl-click opens the page in a new tab | A button | The owner: open a page in a tab of its own, as on any site |
|
||||||
| 2026-10-02 | E31 Plex detail second column (refined) | The facts over the key crew beside the text; the keywords, chips that filter like the genres, under the text; the show page gets all three too | All three in the column, as the mockup drew it | At 20 keywords and 12 crew the column ran to twice the text's height (849 vs ~400 px at 133%): a hole under the text instead of beside it |
|
| 2026-10-02 | E31 Plex detail second column (refined) | The facts over the key crew beside the text; the keywords, chips that filter like the genres, under the text; the show page gets all three too | All three in the column, as the mockup drew it | At 20 keywords and 12 crew the column ran to twice the text's height (849 vs ~400 px at 133%): a hole under the text instead of beside it |
|
||||||
| 2026-10-02 | E31 Plex detail poster and crew | The movie and show poster keeps its 2:3 and grows a rung in a wide hero (w-56 at a 64rem panel); the crew shows its first 6 names and "Show N more" (a list of 8 or fewer shows whole) | Poster stretched to the row; a "More" on the keywords | The row is as tall as the crew column, so the poster drew ~175×590 and cropped; the crew, not the keywords, set the hero's height |
|
| 2026-10-02 | E31 Plex detail poster and crew | The movie and show poster keeps its 2:3 and grows a rung in a wide hero (w-56 at a 64rem panel); the crew shows its first 6 names and "Show N more" (a list of 8 or fewer shows whole) | Poster stretched to the row; a "More" on the keywords | The row is as tall as the crew column, so the poster drew ~175×590 and cropped; the crew, not the keywords, set the hero's height |
|
||||||
|
| 2026-10-04 | E32 `field` focus skips a mode switch | A form opens on its first field; a control above it that is not a field (the composer's Edit / Preview) is passed over by marking the field `data-initial-focus` | The first text field everywhere | That moved the focus in the download dialog off its profile choice and in the upload dialog off its drop zone; the mark changes only the form that needs it |
|
||||||
|
|||||||
+24
-11
@@ -39,36 +39,49 @@ decisions and the chosen patterns, rendered with the app's real CSS. It never re
|
|||||||
## The gate
|
## The gate
|
||||||
|
|
||||||
A change is done when all of this is green. CI runs the same checks server-side
|
A change is done when all of this is green. CI runs the same checks server-side
|
||||||
(`.gitea/workflows/check.yml`), so "main is always green".
|
(`.gitea/workflows/check.yml`), so "main is always green". What each gate deliberately does NOT
|
||||||
|
check — and why — is the [gate register](gates.md); a test fails when a narrowing is missing from it.
|
||||||
|
|
||||||
**Backend**
|
**Backend**
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# lint + format check
|
# lint + format check (ruff is pinned in backend/requirements-dev.txt)
|
||||||
python -m ruff check --config backend/ruff.toml backend scripts
|
python -m ruff check --config backend/ruff.toml backend scripts
|
||||||
python -m ruff format --check --config backend/ruff.toml backend scripts
|
python -m ruff format --check --config backend/ruff.toml backend scripts
|
||||||
|
|
||||||
# tests — pure-logic + the DB-backed integration lane, against an ephemeral Postgres.
|
# type check — in the test image (requirements.lock does not install on Windows); new errors only,
|
||||||
# The backend tree is bind-mounted, so it always tests your working files.
|
# the pre-existing ones are in backend/.basedpyright/baseline.json
|
||||||
|
docker compose -f backend/docker-compose.test.yml run --rm --no-deps --build test \
|
||||||
|
python -m basedpyright --project pyrightconfig.json
|
||||||
|
|
||||||
|
# tests — pure-logic + the DB-backed integration lane, against an ephemeral Postgres, ending with the
|
||||||
|
# coverage total. The backend tree is bind-mounted, so it always tests your working files.
|
||||||
docker compose -f backend/docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from test
|
docker compose -f backend/docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from test
|
||||||
docker compose -f backend/docker-compose.test.yml down -v
|
docker compose -f backend/docker-compose.test.yml down -v
|
||||||
```
|
```
|
||||||
|
|
||||||
The test compose sets `SIFTLODE_TEST_REQUIRE_DB=1`, which turns a would-be silent skip (no DB, or no
|
A missing database or ffmpeg is a hard error, not a skip — so the gate can't print green while the
|
||||||
ffmpeg) into a hard failure — so the gate can't print green while the DB-backed tests were quietly not
|
DB-backed tests were quietly not run (`SIFTLODE_TEST_REQUIRE_DB=0` is the explicit "pure-logic only"
|
||||||
run. Tests that need a database take the `db` fixture (`backend/tests/conftest.py`); the rest are
|
run for a machine without Postgres). Tests that need a database take the `db` fixture
|
||||||
pure-logic and run without one.
|
(`backend/tests/conftest.py`); the rest are pure-logic and run without one.
|
||||||
|
|
||||||
**Frontend** (from `frontend/`)
|
**Frontend** (from `frontend/`)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
node ../scripts/check-node-modules.mjs # node_modules == package-lock.json (else: npm ci)
|
||||||
npm run typecheck # tsc x3 (app / node / e2e)
|
npm run typecheck # tsc x3 (app / node / e2e)
|
||||||
npm run lint # eslint
|
npm run lint # eslint (typed rules included; warnings fail)
|
||||||
npm run format:check # prettier
|
npm run format:check # prettier
|
||||||
npm run knip # unused exports/deps
|
npm run knip # unused files/exports/deps — and unused by what SHIPS (--production)
|
||||||
npm run test # vitest
|
npm run test # vitest (incl. the compiler, size/complexity and gate-register contracts)
|
||||||
|
npx vite build # the production bundle (devOnlyGuard fails it on dev-only code)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Coverage is reported, never enforced: `npm run test -- --coverage`, and the pytest lane's last line.
|
||||||
|
The Playwright suite (`npm run e2e`) runs against the `:8080` stack built from your tree; the release
|
||||||
|
requires it. Before an image is pushed, it is scanned with Trivy (HIGH/CRITICAL with a fix fails;
|
||||||
|
exceptions in `.trivyignore.yaml`, each with a reason and an expiry).
|
||||||
|
|
||||||
## Branching
|
## Branching
|
||||||
|
|
||||||
Trunk-based: branch off `main`, keep the branch gate-green, and land the whole change back to `main`
|
Trunk-based: branch off `main`, keep the branch gate-green, and land the whole change back to `main`
|
||||||
|
|||||||
+136
@@ -0,0 +1,136 @@
|
|||||||
|
# Gate register
|
||||||
|
|
||||||
|
Every place a gate deliberately checks **less** than it could: a rule switched off or narrowed for a
|
||||||
|
scope, a flag left off, a lane that does not run everywhere, an accepted finding. One row each, with
|
||||||
|
the reason and the number measured when the row was written.
|
||||||
|
|
||||||
|
**Why this file exists (E32).** The worst defect E32 found was not in any code a reviewer read: a lint
|
||||||
|
rule had been switched off with 64 real hits counted in its own comment, and the "false positives"
|
||||||
|
beside them were the React Compiler skipping 95 functions without a word. The rule since: **a gate
|
||||||
|
with any real hit is never switched off or narrowed** — fix the sites, or suppress each one inline and
|
||||||
|
name its invariant. Whatever is left narrowed is listed here, and
|
||||||
|
[`frontend/src/lib/gates.contract.test.ts`](../frontend/src/lib/gates.contract.test.ts) reads the
|
||||||
|
**effective** configuration and fails when a narrowing is not on this page. A row here is a decision,
|
||||||
|
not an oversight; a narrowing missing from here is a test failure.
|
||||||
|
|
||||||
|
**The snapshot.** ESLint's effective config for every code file `eslint .` reaches, and the lines
|
||||||
|
that run the lanes (`.gitea/workflows/check.yml`, `backend/docker-compose.test.yml`,
|
||||||
|
`backend/Dockerfile.test`, the npm scripts, `vitest.config.ts`, `.prettierignore`), are committed in
|
||||||
|
[`frontend/src/lib/gates.snapshot.json`](../frontend/src/lib/gates.snapshot.json). The test fails on
|
||||||
|
ANY difference — a rule, an option, a glob that moves one file to another config, a new ignore, a CI
|
||||||
|
flag. So a change to a gate is always a diff of that file in review, never a shape a detector failed
|
||||||
|
to model (E32 review: three rounds of a cleverer detector each found a bypass of the last one's
|
||||||
|
fixes). Regenerate it with `GATE_SNAPSHOT=update npx vitest run src/lib/gates.contract.test.ts`
|
||||||
|
(from `frontend/`), and read its diff as a gate change: a new file only adds its path; anything else
|
||||||
|
needs a reason. A new code file is part of that diff by design.
|
||||||
|
|
||||||
|
**How the register is matched:** each narrowing must appear in a table row below as its
|
||||||
|
`code-quoted` id, in a row whose **Scope** cell names the scope exactly as the test reports it (a cell
|
||||||
|
may list several, separated by ` · `). For ESLint the scopes are the snapshot's config names, and a
|
||||||
|
narrowing is read off each config against `src`: a rule at off or warn, or one src applies that a
|
||||||
|
scope does not (one id per plugin family), a `no-restricted-*` list that leaves out an entry src's
|
||||||
|
declares, an option that lets something through (`allow…`, `ignore…`, `except…`), an unused-disable
|
||||||
|
report that is not an error, and a file no config lints. Ruff's TOML is parsed and held to a
|
||||||
|
whitelist of key paths, pytest.ini to a whitelist of whole lines.
|
||||||
|
|
||||||
|
## ESLint (frontend)
|
||||||
|
|
||||||
|
`npm run lint` is `eslint . --max-warnings 0`, so a rule at **warn** fails the lane exactly like an
|
||||||
|
error (the test holds that exact script). The warn rows are listed so that claim stays checked.
|
||||||
|
A core rule switched off because its typescript-eslint twin replaces it (`no-unused-vars` →
|
||||||
|
`@typescript-eslint/no-unused-vars`) is not listed separately; neither are the core rules
|
||||||
|
typescript-eslint's compatibility layer switches off for TS files because the type checker reports
|
||||||
|
them itself (`no-undef`, `no-redeclare`, `no-dupe-keys`, …), nor the formatting rules
|
||||||
|
`eslint-config-prettier` switches off (Prettier owns formatting and has its own lane).
|
||||||
|
|
||||||
|
| Scope | Narrowed | Why | Measured |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| src | `@typescript-eslint/no-unused-vars` — off | tsc owns it (`noUnusedLocals` + `noUnusedParameters` in every project), so a finding is reported once, by one tool. | — |
|
||||||
|
| src | `react-hooks/exhaustive-deps` — warn | Preset level; `--max-warnings 0` makes it fail. Every suppression names its invariant (`eslint-comments/require-description`). | 0 suppressions (E32 S4: a suppression of this rule drops the whole function from the React Compiler, so the two that were left went) |
|
||||||
|
| src | `react-hooks/incompatible-library` — warn | Preset level; fails under `--max-warnings 0`. VirtualGrid (TanStack Virtual) is the one site, suppressed inline. | 1 |
|
||||||
|
| src | `react-hooks/unsupported-syntax` — warn | Preset level; fails under `--max-warnings 0`. | 0 |
|
||||||
|
| src | `react-refresh/only-export-components` — warn | Fails under `--max-warnings 0`; `allowConstantExport` lets a module export constants beside its component. | 0 |
|
||||||
|
| src | `@typescript-eslint/no-unused-expressions` — narrowed | `allowShortCircuit` + `allowTernary`: `cond && f()` / `a ? f() : g()` side-effect statements are an idiom here. | — |
|
||||||
|
| src | `@typescript-eslint/no-floating-promises` — narrowed | `allowForKnownSafeCalls`: React Query's `invalidateQueries`, `refetchQueries`, `cancelQueries`, `resetQueries`, `prefetchQuery`, `prefetchInfiniteQuery` never reject unless asked to (`throwOnError`), so a bare call is safe by name. | 6 names |
|
||||||
|
| e2e | `@typescript-eslint/no-floating-promises` — narrowed | The same `allowForKnownSafeCalls` list (the typed rules are one object for both scopes). | 6 names |
|
||||||
|
| lib/http.ts | `no-restricted-globals` — narrowed | A network owner (`NETWORK_OWNERS` in eslint.config.js): the one module that may call `fetch` (`send`, which `req` and the public pages go through). Every other network global stays restricted there. | 1 global |
|
||||||
|
| lib/api/core.ts | `no-restricted-globals` — narrowed | A network owner: `uploadReq` is the one `XMLHttpRequest` (upload progress). | 1 global |
|
||||||
|
| lib/messagesSocket.ts | `no-restricted-globals` — narrowed | A network owner: the chat socket is the one `WebSocket`. | 1 global |
|
||||||
|
| src tests · ui tests · lib tests | `no-restricted-globals`, `no-restricted-properties` — off | A test stands a double in for the network (`vi.stubGlobal("fetch", …)`) and may reach it directly. | — |
|
||||||
|
| src tests · ui tests · lib tests | `local/literal-a11y-text`, `local/button-primitive`, `local/input-primitive`, `local/surface-role`, `local/glyph-link` — off | A test renders throwaway markup (a raw button as a focus target, an untranslated label) that is not product UI. | — |
|
||||||
|
| FocusSafeButton.tsx + AnchoredPopover.tsx | `local/focus-safe-disabled` — off | FocusSafeButton: it IS the focus-safe primitive the rule points everyone else to. | — |
|
||||||
|
| FocusSafeButton.tsx + AnchoredPopover.tsx | `local/focus-safe-disabled` — off | AnchoredPopover: the popover's menu row: its native `disabled` is what floating-ui's roving focus reads to skip it. | — |
|
||||||
|
| design-lab | `local/literal-a11y-text` — off | The dev-only design lab (never in the bundle — devOnlyGuard fails the build if it is) holds the owner's working copy, not product copy. | — |
|
||||||
|
| e2e | `react-hooks/*`, `react-refresh/*`, `local/*`, `no-console`, `no-restricted-globals`, `no-restricted-properties`, `no-restricted-syntax` — not applied | Playwright specs drive a browser from Node: no React, no product UI, and they print to the console on purpose. They get the base + typed rules + the playwright rules. | — |
|
||||||
|
| ts configs | `react-hooks/*`, `react-refresh/*`, `local/*`, `no-console`, `no-restricted-globals`, `no-restricted-properties`, `no-restricted-syntax` — not applied | The build configs (vite, vitest, playwright, reactPlugin, devOnlyGuard) run in Node at build time. | — |
|
||||||
|
| js configs | `@typescript-eslint/*`, `react-hooks/*`, `react-refresh/*`, `local/*`, `no-console`, `no-restricted-globals`, `no-restricted-properties`, `no-restricted-syntax`, `no-var`, `prefer-const`, `prefer-rest-params`, `prefer-spread`, `no-array-constructor`, `no-unused-expressions` — not applied | `eslint.config.js` and `postcss.config.js`: plain-JS build config, linted with `js.configs.recommended` only. | — |
|
||||||
|
| repo scripts | `scripts/*.mjs` — not in the eslint or prettier lane; `js.configs.recommended` + Prettier via `src/lib/repoScripts.contract.test.ts` | They sit at the repo root, and an ESLint config cannot reach past its own directory. The contract test lints and format-checks them from the repo root, to the same bar as the `js configs` row; no typed rules, as they are in no tsc project. | 0 lint hits; 2 files reformatted (E32 review) |
|
||||||
|
| everywhere | ignored paths: `dist`, `node_modules`, `playwright-report`, `test-results`, `e2e/.auth` | Build output, installed packages, test artefacts, a live Playwright session. Generated, not written. | — |
|
||||||
|
|
||||||
|
**Not enabled** (decided, not forgotten): `strictTypeChecked` — owner decision E32 D3, ~2000 hits with
|
||||||
|
none real in the sample; the curated typed pair `no-floating-promises` + `no-deprecated` runs instead.
|
||||||
|
|
||||||
|
## TypeScript
|
||||||
|
|
||||||
|
| Scope | Narrowed | Why | Measured |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| config files | `eslint.config.js — not type-checked` | Linted (the `js configs` row) and executed by every lint run, but in no tsc project. Under the node project's `strict` + `checkJs` it reports 43 errors: 42 implicit-`any` parameters in the local rules' visitors (JSX nodes, which ESLint's own `Rule` typings do not model) and one inferred `string` severity. None is a defect; typing them means moving the local rules onto typescript-eslint's `TSESTree` types — a refactor of its own, not done in E32. | 43 (E32 review) |
|
||||||
|
| all three projects | `skipLibCheck` | The `.d.ts` of installed packages are not type-checked (ours are). Standard for an app: a package's own types are its maintainers' gate, and checking them costs minutes and reports errors we cannot fix. | — |
|
||||||
|
|
||||||
|
Every project has `strict`, `noUncheckedIndexedAccess`, `noUnusedLocals`, `noUnusedParameters` and
|
||||||
|
`noFallthroughCasesInSwitch` on; the test fails if one goes off, if a `strict` sub-flag
|
||||||
|
(`strictNullChecks`, `noImplicitAny`, …) is set false beside it, if a project gains an `extends`,
|
||||||
|
`exclude` or `files`, or if its `include` changes — and a config file at the frontend root that no
|
||||||
|
project checks needs a row.
|
||||||
|
|
||||||
|
## Ruff (backend + scripts)
|
||||||
|
|
||||||
|
| Scope | Narrowed | Why | Measured |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| backend + scripts | `B008` — narrowed: `fastapi.Depends`, `fastapi.Query` declared immutable | FastAPI's dependency injection IS a call in a default argument, on every route. Every other call in a default still fails. | 474 sites, all of them FastAPI |
|
||||||
|
| backend/app | `C901` — not in ruff; measured by `tests/test_source_size.py` | Ruff has no per-function baseline, so turning it on would need a `noqa` per offender, and a `noqa` lets a function grow without limit. The test holds each offender, by name, at its current value. Tests and scripts are outside it: see the size / complexity rows under Lanes. | 32 functions over 10 |
|
||||||
|
|
||||||
|
## basedpyright (backend)
|
||||||
|
|
||||||
|
| Scope | Narrowed | Why | Measured |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| backend/app + alembic | `typeCheckingMode` — `standard` | Not `strict`/`all`: those report on every untyped third-party boundary (SQLAlchemy rows, yt-dlp dicts) — thousands of findings, near-zero real ones. `standard` was measured at 7 findings, all fixed (E32 S1). | — |
|
||||||
|
| backend/app + alembic | `baseline.json` — pre-existing errors absorbed | The lane fails on NEW errors only; the baseline only shrinks (the lane fails when a run tightens it and it is not committed). Matching is per (file, rule, column span) + count, so burn the hot files first. | 195 entries (E32 S1) |
|
||||||
|
| tests/ | not type-checked | A test's job is to fail loudly when wrong — the opposite of the silent runtime failure this lane catches — and annotating fixtures for a checker is churn. | — |
|
||||||
|
|
||||||
|
## pytest (backend)
|
||||||
|
|
||||||
|
| Scope | Narrowed | Why | Measured |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| a hand run | `SIFTLODE_TEST_REQUIRE_DB=0` — the DB-backed and ffmpeg tests skip | The explicit "pure-logic only" run for a machine without Postgres. Off by default since E32: without it a missing DB is a hard error; the gate and CI never set it. | — |
|
||||||
|
|
||||||
|
## knip (frontend)
|
||||||
|
|
||||||
|
| Scope | Narrowed | Why | Measured |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| --production | `!**/*.testkit.ts!`, `!e2e/**!` | Test kits and the e2e suite are test code: the production pass asks what the SHIPPED code uses. | — |
|
||||||
|
| --production | `!reactPlugin.ts!`, `!devOnlyGuard.ts!` | Build tooling the vite config imports: it shapes the bundle but is not in it. | — |
|
||||||
|
| --production | `@internal` exports | An export only its test imports says so with a JSDoc `@internal` tag naming the test; knip skips those in production mode. | 35 (E32 S1) |
|
||||||
|
|
||||||
|
## Lanes and pipeline
|
||||||
|
|
||||||
|
| Scope | Narrowed | Why | Measured |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| check | e2e (Playwright) not in `check` | It needs the :8080 stack built from the tree (minutes). It runs in `check-full`, and `release` rebuilds the stack and requires it (E32 D2) — nothing ships without it. | 5 specs × 3 engines |
|
||||||
|
| CI | no e2e, no Trivy | The lab runner has no browser stack, and Trivy's DB download is not reachable from it. Both run in `release` / `publish` on the dev machine, before anything is pushed. | — |
|
||||||
|
| coverage | no threshold | Owner decision E32 D4: reported on both sides, never enforced. "Every regression test seen red" is the bar; a percentage invites tests written to move it. | frontend 80% / backend 78% of lines (E32 S1) |
|
||||||
|
| CVE gate | HIGH/CRITICAL with a fix available only | `--ignore-unfixed` + `--severity HIGH,CRITICAL`: a finding with no fix cannot be acted on by an upgrade; it shows in the scan output. A fixable one fails the publish, and the only way past it is `.trivyignore.yaml`. | 0 open (E32 S1) |
|
||||||
|
| npm audit (CI) | `--omit=dev --audit-level=high` | What ships to the browser, at high or above. The dev toolchain's advisories were burned down by hand in E32 S1 (18 → 0). | 0 |
|
||||||
|
| React Compiler | the functions it skips, pinned | `components/compilerCoverage.contract.test.ts` — the list may only shrink. End state: only VirtualGrid (TanStack Virtual, an IncompatibleLibrary) remains. | 93 (E32 S1) → 29 (E32 S3) → 1, VirtualGrid (E32 S4) |
|
||||||
|
| size / complexity | today's offenders as ceilings | `src/lib/sourceSize.contract.test.ts`, `backend/tests/test_source_size.py`: listed files and functions may not grow, nothing new may cross the limit, an entry back under it must be deleted. A function is held by its name, not its rank in the file. | files >800 lines: 13 FE / 9 BE; functions over the limit: 64 FE / 32 BE (E32 review) |
|
||||||
|
| size / complexity | shipped code only — not tests, test kits, e2e or `scripts/` | The ratchet guards what runs in production. A test file is long because it is a list of cases, and a test kit is test code; splitting either moves lines without making anything easier to change. `scripts/` is release and seed tooling, not in the image. | outside it: tests >800 lines 2 FE / 5 BE; functions over the limit 2 FE test code (`gates.contract.test.ts` `eslintNarrowings` 23, E32 review round 3; `compilerSkips.testkit.ts` 25), 1 BE test (11), 1 script (`e2e_seed.py` `main` 13) |
|
||||||
|
|
||||||
|
## Accepted findings
|
||||||
|
|
||||||
|
Entries in `.trivyignore.yaml` (CVE gate) and `--ignore-vuln` flags on CI's `pip-audit` step. Each
|
||||||
|
must have a row here (Scope `accepted`) with its reason and its expiry; the test fails on an entry
|
||||||
|
without one, and on a `.trivyignore.yaml` entry with no `statement` or no `expired_at`.
|
||||||
|
|
||||||
|
| Scope | Finding | Why | Expires |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| — | none | — | — |
|
||||||
+13
-8
@@ -13,19 +13,24 @@ export async function gotoPage(page: Page, id: string): Promise<void> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Click a channel link that is currently WITHIN the viewport — what a real user does. Never use
|
* Click a channel link a real user can actually SEE and hit. Never use `.first()` for this after
|
||||||
* `.first()` for this after scrolling: the first rendered card lives in the virtualizer's overscan
|
* scrolling: the first rendered card lives in the virtualizer's overscan ABOVE the viewport, so
|
||||||
* ABOVE the viewport, so Playwright would scroll it into view to click it, moving the scroll
|
* Playwright would scroll it into view to click it, moving the scroll position — fatal to any test
|
||||||
* position — fatal to any test that then asserts on where the feed was. Returns false if none found.
|
* that then asserts on where the feed was. Being inside the viewport is not enough either: a card
|
||||||
|
* can sit under the page chrome above the scroller (the update banner, the toolbar band), and the
|
||||||
|
* covered click makes Playwright scroll and retry — the same move (E32 S2: webkit failed 3 in 24).
|
||||||
|
* So the link must be the element hit at its own centre. Returns false if none found.
|
||||||
*/
|
*/
|
||||||
export async function clickVisibleChannelLink(page: Page): Promise<boolean> {
|
export async function clickVisibleChannelLink(page: Page): Promise<boolean> {
|
||||||
const links = page.getByTestId("video-card-channel").filter({ visible: true });
|
const links = page.getByTestId("video-card-channel").filter({ visible: true });
|
||||||
const viewport = page.viewportSize();
|
|
||||||
const height = viewport?.height ?? 800;
|
|
||||||
const count = await links.count();
|
const count = await links.count();
|
||||||
for (let i = 0; i < count; i++) {
|
for (let i = 0; i < count; i++) {
|
||||||
const box = await links.nth(i).boundingBox();
|
const hittable = await links.nth(i).evaluate((el) => {
|
||||||
if (box && box.y >= 0 && box.y + box.height <= height) {
|
const r = el.getBoundingClientRect();
|
||||||
|
const hit = document.elementFromPoint(r.left + r.width / 2, r.top + r.height / 2);
|
||||||
|
return hit !== null && el.contains(hit);
|
||||||
|
});
|
||||||
|
if (hittable) {
|
||||||
await links.nth(i).click();
|
await links.nth(i).click();
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
+22
-12
@@ -6,18 +6,12 @@ import { openApp } from "./helpers";
|
|||||||
// this exercises is precisely what the module-registry refactor replaces.
|
// this exercises is precisely what the module-registry refactor replaces.
|
||||||
test.describe("page switching", () => {
|
test.describe("page switching", () => {
|
||||||
// Core + admin modules (the E2E account is an admin, matching the real primary user), so this
|
// Core + admin modules (the E2E account is an admin, matching the real primary user), so this
|
||||||
// also exercises the render ternary's admin branches (scheduler / config / users / audit).
|
// also exercises the admin pages. Since E31 those sit behind ONE rail row ("Administration",
|
||||||
const RAIL = [
|
// `nav-admin`) that opens a panel of them — the spec still listed them as rail rows of their own,
|
||||||
"channels",
|
// and it failed on every run from E31 to E32 without anyone seeing it, because e2e was in no
|
||||||
"playlists",
|
// required gate (it is now part of `release`).
|
||||||
"stats",
|
const RAIL = ["channels", "playlists", "stats", "settings", "feed"];
|
||||||
"scheduler",
|
const ADMIN = ["scheduler", "config", "users", "audit"];
|
||||||
"config",
|
|
||||||
"users",
|
|
||||||
"audit",
|
|
||||||
"settings",
|
|
||||||
"feed",
|
|
||||||
];
|
|
||||||
|
|
||||||
test("each rail item activates its page", async ({ page }) => {
|
test("each rail item activates its page", async ({ page }) => {
|
||||||
await openApp(page);
|
await openApp(page);
|
||||||
@@ -27,6 +21,22 @@ test.describe("page switching", () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("each admin page opens from the Administration panel and marks it current", async ({
|
||||||
|
page,
|
||||||
|
}) => {
|
||||||
|
await openApp(page);
|
||||||
|
for (const id of ADMIN) {
|
||||||
|
await page.getByTestId("nav-admin").click();
|
||||||
|
await page.getByTestId(`nav-admin-${id}`).click();
|
||||||
|
// The panel closes on a pick; the group row carries the current page…
|
||||||
|
await expect(page.getByTestId("nav-admin")).toHaveAttribute("aria-current", "page");
|
||||||
|
// …and, reopened, the panel marks which one.
|
||||||
|
await page.getByTestId("nav-admin").click();
|
||||||
|
await expect(page.getByTestId(`nav-admin-${id}`)).toHaveAttribute("aria-current", "page");
|
||||||
|
await page.keyboard.press("Escape");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
test("browser Back steps through visited pages", async ({ page }) => {
|
test("browser Back steps through visited pages", async ({ page }) => {
|
||||||
await openApp(page);
|
await openApp(page);
|
||||||
|
|
||||||
|
|||||||
@@ -11,6 +11,8 @@
|
|||||||
"noUnusedLocals": true,
|
"noUnusedLocals": true,
|
||||||
"noUnusedParameters": true,
|
"noUnusedParameters": true,
|
||||||
"noFallthroughCasesInSwitch": true,
|
"noFallthroughCasesInSwitch": true,
|
||||||
|
// E32: an index read is `T | undefined` here, as it really is (0 errors when switched on).
|
||||||
|
"noUncheckedIndexedAccess": true,
|
||||||
"esModuleInterop": true,
|
"esModuleInterop": true,
|
||||||
"resolveJsonModule": true
|
"resolveJsonModule": true
|
||||||
},
|
},
|
||||||
|
|||||||
+129
-26
@@ -4,16 +4,25 @@ import reactHooks from "eslint-plugin-react-hooks";
|
|||||||
import reactRefresh from "eslint-plugin-react-refresh";
|
import reactRefresh from "eslint-plugin-react-refresh";
|
||||||
import prettier from "eslint-config-prettier";
|
import prettier from "eslint-config-prettier";
|
||||||
import globals from "globals";
|
import globals from "globals";
|
||||||
|
import eslintComments from "@eslint-community/eslint-plugin-eslint-comments";
|
||||||
|
import vitest from "@vitest/eslint-plugin";
|
||||||
|
import playwright from "eslint-plugin-playwright";
|
||||||
|
|
||||||
// Flat config (ESLint 9). Deliberately NARROW: the high-value rule here is
|
// Flat config (ESLint 9). Deliberately NARROW: the high-value rules here are react-hooks' (the
|
||||||
// react-hooks/exhaustive-deps — 33 suppressions across the app exist for it, and until this
|
// hook rules and the React Compiler's diagnostics). The 33 inline `exhaustive-deps` suppressions this
|
||||||
// sprint nothing ran to make them mean anything. tsc already owns unused vars, undefined names
|
// config first met are all gone (0 left, E32 review); what is still suppressed is suppressed inline,
|
||||||
|
// one site at a time, each naming its invariant. tsc already owns unused vars, undefined names
|
||||||
// and types (noUnusedLocals/Parameters + strict), so typescript-eslint's non-type-checked
|
// and types (noUnusedLocals/Parameters + strict), so typescript-eslint's non-type-checked
|
||||||
// `recommended` is layered on but its overlap with tsc is turned off below to avoid double-reporting
|
// `recommended` is layered on but its overlap with tsc is turned off below to avoid double-reporting
|
||||||
// and a churny diff. `eslint-config-prettier` is LAST so formatting rules never fight Prettier.
|
// and a churny diff. `eslint-config-prettier` is LAST so formatting rules never fight Prettier.
|
||||||
//
|
//
|
||||||
// reportUnusedDisableDirectives: a `// eslint-disable` that no longer suppresses anything is itself
|
// reportUnusedDisableDirectives: a `// eslint-disable` that no longer suppresses anything is itself
|
||||||
// an error — that is what keeps inert disable comments from re-accumulating (the S1b mandate).
|
// an error — that is what keeps inert disable comments from re-accumulating (the S1b mandate).
|
||||||
|
//
|
||||||
|
// E32 (gate integrity): every switched-off or narrowed rule here is listed, with its reason and its
|
||||||
|
// measured count, in docs/gates.md, and lib/gates.contract.test.ts fails when one is not. A rule
|
||||||
|
// with a single real hit is never switched off — its sites are fixed, or each suppressed inline with
|
||||||
|
// a `-- <the invariant>` description (eslint-comments/require-description makes that mandatory).
|
||||||
|
|
||||||
// E22 S4: every way out of the browser goes through the one module that LOGS it — a request line
|
// E22 S4: every way out of the browser goes through the one module that LOGS it — a request line
|
||||||
// with a request id (lib/http.ts), the upload XHR (lib/api/core.ts), the messages socket
|
// with a request id (lib/http.ts), the upload XHR (lib/api/core.ts), the messages socket
|
||||||
@@ -472,47 +481,103 @@ const SPLIT_CLASS = "/^([a-z][\\w-]*:[\\w\\[!-]|:[a-z\\[!])/";
|
|||||||
const SPLIT_CLASS_MESSAGE =
|
const SPLIT_CLASS_MESSAGE =
|
||||||
"a class split by an interpolation (`${x}hover:…`): Tailwind never sees it whole and emits no CSS; write the class literally (a shared guard is the `act:` variant).";
|
"a class split by an interpolation (`${x}hover:…`): Tailwind never sees it whole and emits no CSS; write the class literally (a shared guard is the `act:` variant).";
|
||||||
|
|
||||||
|
// Applies to EVERY linted file: a suppression says which rule and why. A bare
|
||||||
|
// `eslint-disable-next-line rule` named nothing in four places the E32 audit found, against the
|
||||||
|
// CLAUDE.md rule that each one names its invariant; a blanket `eslint-disable` names no rule at all.
|
||||||
|
const suppressionRules = {
|
||||||
|
plugins: { "@eslint-community/eslint-comments": eslintComments },
|
||||||
|
linterOptions: { reportUnusedDisableDirectives: "error" },
|
||||||
|
rules: {
|
||||||
|
"@eslint-community/eslint-comments/require-description": ["error", { ignore: [] }],
|
||||||
|
"@eslint-community/eslint-comments/no-unlimited-disable": "error",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
// The two type-aware rules (E32 D3: a curated pair, not strictTypeChecked — ~2000 hits there, none
|
||||||
|
// real in the sample). A floating promise is a rejection nobody handles: the audit found two real
|
||||||
|
// ones (ChatDock's device key store, VideoEditor's play()). `void` marks a deliberate fire-and-forget
|
||||||
|
// whose callee reports its own failure. React Query's invalidate/refetch family never rejects
|
||||||
|
// unless asked to (`throwOnError`), so those calls are known-safe by name.
|
||||||
|
const typedRules = {
|
||||||
|
"@typescript-eslint/no-floating-promises": [
|
||||||
|
"error",
|
||||||
|
{
|
||||||
|
allowForKnownSafeCalls: [
|
||||||
|
{
|
||||||
|
from: "package",
|
||||||
|
package: "@tanstack/query-core",
|
||||||
|
name: [
|
||||||
|
"invalidateQueries",
|
||||||
|
"refetchQueries",
|
||||||
|
"cancelQueries",
|
||||||
|
"resetQueries",
|
||||||
|
"prefetchQuery",
|
||||||
|
"prefetchInfiniteQuery",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"@typescript-eslint/no-deprecated": "error",
|
||||||
|
};
|
||||||
|
|
||||||
export default tseslint.config(
|
export default tseslint.config(
|
||||||
{ ignores: ["dist", "node_modules", "playwright-report", "test-results", "e2e/.auth"] },
|
{ ignores: ["dist", "node_modules", "playwright-report", "test-results", "e2e/.auth"] },
|
||||||
|
suppressionRules,
|
||||||
{
|
{
|
||||||
files: ["src/**/*.{ts,tsx}"],
|
files: ["src/**/*.{ts,tsx}"],
|
||||||
extends: [js.configs.recommended, ...tseslint.configs.recommended],
|
extends: [js.configs.recommended, ...tseslint.configs.recommended],
|
||||||
languageOptions: {
|
languageOptions: {
|
||||||
ecmaVersion: 2022,
|
ecmaVersion: 2022,
|
||||||
globals: { ...globals.browser },
|
globals: { ...globals.browser },
|
||||||
|
parserOptions: { projectService: true, tsconfigRootDir: import.meta.dirname },
|
||||||
},
|
},
|
||||||
plugins: {
|
plugins: {
|
||||||
"react-hooks": reactHooks,
|
"react-hooks": reactHooks,
|
||||||
"react-refresh": reactRefresh,
|
"react-refresh": reactRefresh,
|
||||||
local: localRules,
|
local: localRules,
|
||||||
},
|
},
|
||||||
linterOptions: {
|
|
||||||
reportUnusedDisableDirectives: "error",
|
|
||||||
},
|
|
||||||
rules: {
|
rules: {
|
||||||
...reactHooks.configs.recommended.rules,
|
...reactHooks.configs.recommended.rules,
|
||||||
|
...typedRules,
|
||||||
|
// The React Compiler's own diagnostics that `recommended` leaves out (E32). Each is a function
|
||||||
|
// the compiler SKIPS, silently — `panicThreshold` is "none" and there is no logger, so a lint
|
||||||
|
// error is the only place the skip is said out loud. `todo` is the compiler's "cannot lower
|
||||||
|
// this shape yet" (E32 S4 rewrote all 31 try statements it found); the rest had 0 hits when
|
||||||
|
// switched on. Most are tripwires, but three cannot report anything under
|
||||||
|
// eslint-plugin-react-hooks 7.1.1 (E32 review, read in its source and probed):
|
||||||
|
// `exhaustive-effect-dependencies` (its validation is compiled off — a missing effect dependency
|
||||||
|
// is reported by the classic `exhaustive-deps` only), `memoized-effect-dependencies` (nothing
|
||||||
|
// emits its category) and `rule-suppression` (lint mode hands it no suppression rules, so an
|
||||||
|
// inline disable is never named). They stay on for the plugin version that wires them up; a
|
||||||
|
// function the compiler skips over a suppression is caught by the coverage ratchet below.
|
||||||
|
// `todo` does NOT see every such shape — a conditional, a loop
|
||||||
|
// or an optional call inside a try skips the function with no lint hit at all — so the full
|
||||||
|
// per-function picture is components/compilerCoverage.contract.test.ts.
|
||||||
|
"react-hooks/todo": "error",
|
||||||
|
"react-hooks/memo-dependencies": "error",
|
||||||
|
"react-hooks/rule-suppression": "error",
|
||||||
|
"react-hooks/invariant": "error",
|
||||||
|
"react-hooks/syntax": "error",
|
||||||
|
"react-hooks/void-use-memo": "error",
|
||||||
|
"react-hooks/component-hook-factories": "error",
|
||||||
|
"react-hooks/capitalized-calls": "error",
|
||||||
|
"react-hooks/hooks": "error",
|
||||||
|
"react-hooks/memoized-effect-dependencies": "error",
|
||||||
|
"react-hooks/exhaustive-effect-dependencies": "error",
|
||||||
|
"react-hooks/no-deriving-state-in-effects": "error",
|
||||||
// eslint-plugin-react-hooks 7's `recommended` is no longer two rules — it carries the whole
|
// eslint-plugin-react-hooks 7's `recommended` is no longer two rules — it carries the whole
|
||||||
// React Compiler rule set (E27 S1). On this codebase that landed 283 findings, and they were
|
// React Compiler rule set (E27 S1). On this codebase that landed 283 findings. 66 were fixed or
|
||||||
// triaged rather than bulk-suppressed, because the two answers are different:
|
// suppressed inline, each naming its invariant; there is DELIBERATELY no suppressions file — a
|
||||||
|
// bulk `--suppress-all` would hide the next violation instead of answering it.
|
||||||
//
|
//
|
||||||
// react-hooks/refs 217 — OFF, see below.
|
// `refs` (the other 217) was switched OFF in E27 as mostly false positives, and that triage was
|
||||||
// the other 66 — real. They went into eslint-suppressions.json as a ratchet for one
|
// wrong: the rule and the React Compiler share one analysis, so every hit was a function the
|
||||||
// sprint and E27 S2 burned the file down to nothing: what survives is
|
// compiler SKIPS — 21 popover components among them, and Channels' 0.9 s row menu was the bill
|
||||||
// an inline directive at the site, each naming the invariant it
|
// (E32). E32 S3–S4 fixed every site (245 measured) instead of narrowing the rule: the popover
|
||||||
// protects. There is DELIBERATELY no suppressions file any more — a
|
// engine is read through destructuring, render-phase ref writes became `useEffectEvent` /
|
||||||
// bulk `--suppress-all` would hide the next violation instead of
|
// `useLatest`. It is on, at error, with no exception.
|
||||||
// answering it, and it cannot hold a warning-level rule anyway.
|
"react-hooks/refs": "error",
|
||||||
//
|
|
||||||
// `refs` is off because it fires on this app's popover architecture, not on its history, so
|
|
||||||
// suppressing it would not help the next popover anyone writes. Measured over the 217:
|
|
||||||
// 79 are `pop.refs.setReference` / `getReferenceProps` — @floating-ui/react's callback-ref
|
|
||||||
// setters, which are the documented way to use it — and 74 more are passing the popover handle
|
|
||||||
// down as a prop (`popover={pop}`, `ref={fade.ref}`), which is what `components/ui`'s popover
|
|
||||||
// engine is FOR. That is 153 of 217 flagged for having the word `refs` in an object that is not
|
|
||||||
// a React ref. The remaining ~64 are genuine render-phase `someRef.current = x` writes and are
|
|
||||||
// real (E27 S2 answered the ones the other rules could see, at their sites).
|
|
||||||
// Re-evaluate when the rule can: it shipped in 1.0.0 and this looks like over-approximation.
|
|
||||||
"react-hooks/refs": "off",
|
|
||||||
"local/jsx-comment-text": "error",
|
"local/jsx-comment-text": "error",
|
||||||
"local/focus-safe-disabled": "error",
|
"local/focus-safe-disabled": "error",
|
||||||
"local/button-primitive": "error",
|
"local/button-primitive": "error",
|
||||||
@@ -709,7 +774,12 @@ export default tseslint.config(
|
|||||||
// Tests stand doubles in for the network (`vi.stubGlobal("fetch", …)`) and may reach it directly.
|
// Tests stand doubles in for the network (`vi.stubGlobal("fetch", …)`) and may reach it directly.
|
||||||
// …and render throwaway markup whose labels no user ever hears.
|
// …and render throwaway markup whose labels no user ever hears.
|
||||||
files: ["src/**/*.test.{ts,tsx}"],
|
files: ["src/**/*.test.{ts,tsx}"],
|
||||||
|
plugins: { vitest },
|
||||||
rules: {
|
rules: {
|
||||||
|
// A focused test silently skips the rest of the suite; a skipped one is a test nobody runs
|
||||||
|
// (E32 — vitest's allowOnly is on outside CI, so nothing else catches a stray `.only` locally).
|
||||||
|
"vitest/no-focused-tests": "error",
|
||||||
|
"vitest/no-disabled-tests": "error",
|
||||||
"no-restricted-globals": "off",
|
"no-restricted-globals": "off",
|
||||||
"no-restricted-properties": "off",
|
"no-restricted-properties": "off",
|
||||||
"local/literal-a11y-text": "off",
|
"local/literal-a11y-text": "off",
|
||||||
@@ -720,5 +790,38 @@ export default tseslint.config(
|
|||||||
"local/glyph-link": "off",
|
"local/glyph-link": "off",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
// Outside src (E32): the e2e suite and the TypeScript build-tool configs went unlinted — the
|
||||||
|
// flat config named only src/**, so `eslint .` skipped them without a word. The same base and
|
||||||
|
// typed rules as the app; Node globals, since none of it runs in the browser.
|
||||||
|
files: ["e2e/**/*.ts", "*.config.ts", "reactPlugin.ts", "devOnlyGuard.ts"],
|
||||||
|
extends: [js.configs.recommended, ...tseslint.configs.recommended],
|
||||||
|
languageOptions: {
|
||||||
|
ecmaVersion: 2022,
|
||||||
|
globals: { ...globals.node },
|
||||||
|
parserOptions: { projectService: true, tsconfigRootDir: import.meta.dirname },
|
||||||
|
},
|
||||||
|
rules: {
|
||||||
|
...typedRules,
|
||||||
|
// tsc owns these here too (both tsconfig.node.json and e2e/tsconfig.json are strict +
|
||||||
|
// noUnusedLocals/Parameters).
|
||||||
|
"@typescript-eslint/no-unused-vars": "off",
|
||||||
|
"no-undef": "off",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
files: ["e2e/**/*.ts"],
|
||||||
|
plugins: { playwright },
|
||||||
|
rules: {
|
||||||
|
"playwright/no-focused-test": "error",
|
||||||
|
"playwright/no-skipped-test": "error",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// The two plain-JS configs: this file and postcss's.
|
||||||
|
files: ["*.config.js"],
|
||||||
|
extends: [js.configs.recommended],
|
||||||
|
languageOptions: { ecmaVersion: 2022, sourceType: "module", globals: { ...globals.node } },
|
||||||
|
},
|
||||||
prettier,
|
prettier,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
{
|
|
||||||
"$schema": "https://unpkg.com/knip@6/schema.json",
|
|
||||||
"playwright": {
|
|
||||||
"config": ["playwright.config.ts"],
|
|
||||||
"entry": ["e2e/**/*.@(spec|setup).ts"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user