Compare commits
131
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aff5a9922c | ||
|
|
cace9a004d | ||
|
|
bfc104a9b4 | ||
|
|
0340d0981c | ||
|
|
18edbecb1e | ||
|
|
8ffa65a9da | ||
|
|
f54dd98129 | ||
|
|
44d4e4876d | ||
|
|
e4ac061721 | ||
|
|
54ba92bb05 | ||
|
|
472cefed55 | ||
|
|
e764ca4568 | ||
|
|
296af2a498 | ||
|
|
66c00327b1 | ||
|
|
31879b01f4 | ||
|
|
15a59f4695 | ||
|
|
0653203d81 | ||
|
|
4ce0a702f4 | ||
|
|
9845c8d0f7 | ||
|
|
7c431d73fb | ||
|
|
67d32fecd6 | ||
|
|
f39c5bb25c | ||
|
|
367bcebc9f | ||
|
|
429fbde866 | ||
|
|
4c013595b5 | ||
|
|
b605c28b32 | ||
|
|
cee0bc95b6 | ||
|
|
5c6172754a | ||
|
|
3575237382 | ||
|
|
14913a6447 | ||
|
|
9fb4db2546 | ||
|
|
2bfe7d53eb | ||
|
|
4169cfe4ff | ||
|
|
030ba7297a | ||
|
|
c0cda545e0 | ||
|
|
2ab188d873 | ||
|
|
e7b5b8e87f | ||
|
|
5f09259bbf | ||
|
|
c361fc2581 | ||
|
|
81827fcf28 | ||
|
|
4e97fc1a84 | ||
|
|
02a2102523 | ||
|
|
704c6c35d4 | ||
|
|
c959859d29 | ||
|
|
072d8fc88f | ||
|
|
87ef075ea7 | ||
|
|
8b8c077a29 | ||
|
|
6465f0ce75 | ||
|
|
ee356f8a1f | ||
|
|
588a1a82a1 | ||
|
|
80ea1918e3 | ||
|
|
83326ecf53 | ||
|
|
1d6c5f051e | ||
|
|
a702e72973 | ||
|
|
d0f5559c49 | ||
|
|
5134ba5433 | ||
|
|
c51df4e0b9 | ||
|
|
f9681deecf | ||
|
|
73a58674cf | ||
|
|
e008c48284 | ||
|
|
f8df504d18 | ||
|
|
59d50615c4 | ||
|
|
ecbf5737b6 | ||
|
|
f38833f104 | ||
|
|
9105c4db3d | ||
|
|
ee2bd81254 | ||
|
|
48285a715b | ||
|
|
e3ee1baa94 | ||
|
|
a7b4b2ed79 | ||
|
|
7dc9f712a0 | ||
|
|
0842babc77 | ||
|
|
5720356191 | ||
|
|
2cc77757d9 | ||
|
|
a342e15e05 | ||
|
|
be738d3791 | ||
|
|
95aa57a2ff | ||
|
|
b4dae04631 | ||
|
|
7ee0e9cd02 | ||
|
|
ea7bf593cb | ||
|
|
19f1905da9 | ||
|
|
5838cf1c56 | ||
|
|
bb2d6999fd | ||
|
|
a26d96630d | ||
|
|
b52c87b336 | ||
|
|
6de6dece03 | ||
|
|
ca7e4b16a0 | ||
|
|
003043a989 | ||
|
|
80cdafdbfe | ||
|
|
9fdf274422 | ||
|
|
07e95ca8e8 | ||
|
|
dd2d53bb96 | ||
|
|
b57aec113e | ||
|
|
42d0173095 | ||
|
|
f26e329ecf | ||
|
|
579af9cf89 | ||
|
|
61ee7af9e4 | ||
|
|
61e848e63a | ||
|
|
ed1a2b7256 | ||
|
|
735d08c91d | ||
|
|
e6688edbcc | ||
|
|
726c382524 | ||
|
|
9917d908eb | ||
|
|
b57e4fb9bc | ||
|
|
ca9dd54923 | ||
|
|
837fdcd408 | ||
|
|
665bc6f7b5 | ||
|
|
5e7169e064 | ||
|
|
002ef0ab84 | ||
|
|
6062007947 | ||
|
|
ccd7170a20 | ||
|
|
02c8c71c53 | ||
|
|
45bd588af7 | ||
|
|
e4bdc5ea6a | ||
|
|
440bdafa74 | ||
|
|
7d9da80795 | ||
|
|
f3b95ed543 | ||
|
|
8ff28dcc69 | ||
|
|
92ef6116f0 | ||
|
|
cba992379c | ||
|
|
f3fce67e4c | ||
|
|
b5f889aa33 | ||
|
|
e025e0f719 | ||
|
|
2a33fc0d42 | ||
|
|
222edaf99e | ||
|
|
01ded15bb4 | ||
|
|
7928d81116 | ||
|
|
ee8aa72abe | ||
|
|
2cd02eb1af | ||
|
|
883e8515b9 | ||
|
|
d4dca46d9f | ||
|
|
8e4dee1bb6 |
@@ -0,0 +1,69 @@
|
||||
# Server-side mirror of the local `siftlode check` gate (E1 S3 / DX-1), so `main` is always green
|
||||
# independent of any one machine. Two parallel jobs match the two local lanes:
|
||||
# frontend — tsc (3 projects) + eslint + prettier --check + knip + vitest
|
||||
# backend — ruff + the full pytest suite (pure-logic + the DB-backed lane) against a real Postgres
|
||||
#
|
||||
# Runner: the lab act_runner (LXC 115) mounts the Docker socket, so service containers work. The backend
|
||||
# runs pytest NATIVELY against a `postgres` service (not the local docker-compose.test.yml, whose
|
||||
# bind-mount is unreliable under a mounted-socket runner); fidelity is kept by installing the SAME pinned
|
||||
# requirements, Postgres 16, and ffmpeg, and by setting SIFTLODE_TEST_REQUIRE_DB=1 so a DB/ffmpeg that
|
||||
# failed to come up is a hard error, never a silent skip. node 22 matches the prod build image
|
||||
# (Dockerfile `node:22-alpine`).
|
||||
name: gate
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
frontend:
|
||||
runs-on: ubuntu-latest
|
||||
container: node:22-bookworm
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: npm ci
|
||||
- run: npm run typecheck
|
||||
- run: npm run lint
|
||||
- run: npm run format:check
|
||||
- run: npm run knip
|
||||
- run: npm run test
|
||||
|
||||
backend:
|
||||
runs-on: ubuntu-latest
|
||||
# A combined python+node image: python 3.12 (matches the prod Dockerfile) for the tests, and node
|
||||
# so the JS actions (checkout) run. actions/setup-python is NOT usable here — the runner can't reach
|
||||
# GitHub's python-versions download, so it must be baked into the image.
|
||||
container: nikolaik/python-nodejs:python3.12-nodejs20
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_USER: siftlode
|
||||
POSTGRES_PASSWORD: siftlode
|
||||
POSTGRES_DB: siftlode_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U siftlode -d siftlode_test"
|
||||
--health-interval 2s
|
||||
--health-timeout 3s
|
||||
--health-retries 30
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg://siftlode:siftlode@postgres:5432/siftlode_test
|
||||
SIFTLODE_TEST_REQUIRE_DB: "1"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# ffmpeg for the poster/thumbnail tests; postgresql-client for the readiness wait below.
|
||||
- run: apt-get update && apt-get install -y --no-install-recommends ffmpeg postgresql-client
|
||||
- name: Wait for Postgres
|
||||
run: until pg_isready -h postgres -U siftlode -d siftlode_test; do sleep 1; done
|
||||
- run: pip install --no-cache-dir -r backend/requirements.lock -r backend/requirements-dev.txt
|
||||
- name: ruff
|
||||
run: python -m ruff check --config backend/ruff.toml backend scripts
|
||||
- name: pytest
|
||||
run: python -m pytest
|
||||
working-directory: backend
|
||||
# Dependency CVE audit over the pinned lock. Non-blocking for now (informational) — a real CVE
|
||||
# shows up in the run without failing the gate; tighten to a hard fail once the noise is known.
|
||||
- name: pip-audit (advisory)
|
||||
run: pip install --no-cache-dir pip-audit && pip-audit -r backend/requirements.lock
|
||||
continue-on-error: true
|
||||
@@ -0,0 +1,83 @@
|
||||
# CLAUDE.md — working agreement for AI agents on Siftlode
|
||||
|
||||
This repository is **fully AI-authored** — every line was written by an AI agent, and it continues to
|
||||
be developed that way. This file is the rules of the road: read it before you touch anything. It is
|
||||
loaded into every agent session, so keep it **short and true**. A rule here that has gone stale is
|
||||
worse than no rule — if you change how the code works, change this file in the same commit.
|
||||
|
||||
Siftlode is a self-hosted YouTube reader (+ a local Plex browser/player and a download center). It is a
|
||||
**modular monolith**: one Postgres, one FastAPI process (uvicorn + APScheduler), one worker process
|
||||
(yt-dlp/ffmpeg), one PO-token sidecar. React 18 + react-query frontend with a router-free navigation
|
||||
provider.
|
||||
|
||||
## The gate — never merge or ship red
|
||||
|
||||
The local gate is **ruff + typecheck (3 tsc projects) + eslint + prettier --check + knip + vitest +
|
||||
pytest**. It runs on every `publish`, which **refuses to build on failure — there is deliberately no
|
||||
`--skip` flag.** Run it and get it green **before** any merge, release, or handing work to review.
|
||||
|
||||
- Full gate (agents with the private ops dispatcher): `siftlode check`.
|
||||
- Directly, without the dispatcher: `cd frontend && npm run typecheck && npm run lint && npm run knip && npm run test`, plus backend `ruff check backend` and `pytest` (the DB-lane needs Postgres).
|
||||
- New DB-backed test? It must actually run, not silently skip — assume `SIFTLODE_TEST_REQUIRE_DB=1`.
|
||||
|
||||
## Workflow
|
||||
|
||||
- **Trunk model.** One long-lived branch, `main`. Branch off it (`feature/`, `bug/`, `improvement/`),
|
||||
keep commits logical and Conventional-Commit-styled.
|
||||
- **No AI attribution in commits.** No `Co-Authored-By` trailer, no "generated by" lines. (This file
|
||||
being AI-written is the intended, visible exception; commits are not.)
|
||||
- **Release is a deliberate, user-owned step** (`siftlode release` from the feature branch: merge to
|
||||
`main` → publish → deploy → verify `/api/version`). Do not self-ship; bring work to a green,
|
||||
committed state and stop for the user's go-ahead.
|
||||
- **Public repo.** Keep ops/topology/secrets out — those live in a separate private repo. Nothing here
|
||||
should assume or leak them.
|
||||
|
||||
## Architecture shape (don't fight it)
|
||||
|
||||
`routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`,
|
||||
`downloads/`, `youtube/` (shared by routes + scheduler + worker) → flat core (`auth`, `quota`,
|
||||
`sysconfig`, `state`, `audit`, `notifications`, `userscope`, `realtime`, `ratelimit`, `progress`,
|
||||
`security`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.**
|
||||
Frontend: `main.tsx` provider tree → `App.tsx` shell → `moduleRegistry` route-chunks → components; data
|
||||
via react-query (`qk` factory), per-account `localStorage`, and server prefs; API through `lib/api/*`
|
||||
slices over `core.req<T>`.
|
||||
|
||||
**Not wanted** (reviewed and rejected as over-engineering for this size): microservices, event bus,
|
||||
CQRS, a repository/DI framework, a router library, a generic CRUD-router factory or base serializer.
|
||||
|
||||
## Canonical patterns — when you have a choice, pick THIS
|
||||
|
||||
| Area | Use | Not |
|
||||
|---|---|---|
|
||||
| ORM | SQLAlchemy 2.0 `select()` | `db.query(...)` |
|
||||
| Request body | a Pydantic model | `payload: dict` |
|
||||
| Ownership check | `owned_or_404` / `get_or_404` | ad-hoc `_own_playlist`/`_owned`/`_own_job` |
|
||||
| Admin gate import | `app.auth` | the `app.routes.admin` re-export |
|
||||
| Upstream failure status | `422` with JSON detail | `502` (reserve 502 for raw byte-proxies) |
|
||||
| User-facing error | structured `detail` (`{code, reason}`) | a bare English string under a bilingual UI |
|
||||
| Mutation response | `{"ok": true, …}` (`{"deleted": id}`) | ad-hoc `{"unsubscribed"}`/`{"cleared": n}`/echo |
|
||||
| Transaction commit | the handler/job-wrapper commits; helpers `flush` | helpers that commit (only `quota` owns its session) |
|
||||
| Admin knob storage | `sysconfig` (values) | env / `app_state` / `scheduler_settings` scatter |
|
||||
| Job status word | the `downloads/status.py` StrEnum | loose `"canceled"`/`"cancelled"` literals |
|
||||
| Current time | inline `datetime.now(timezone.utc)` | new `now_utc`/`_now` helpers |
|
||||
| FE per-account state | `usePref` | raw `readAccount`/`writeAccount` or bespoke hooks |
|
||||
| FE control | the chip/segmented primitives (`Select` only for >6 options) | a native `<select>` |
|
||||
| FE loading/empty/error | `QueryState` / `StateMessage` | ad-hoc conditionals |
|
||||
|
||||
## House rules that bite (learned from real bugs)
|
||||
|
||||
- **A rule stated in a comment must also be enforced in code.** The worst bugs here came from a comment
|
||||
and the code/migration drifting apart (a `cancelled`≠`canceled` literal killed a cancel path). Put the
|
||||
invariant in an enum/constant/shared function, not in prose.
|
||||
- **Fix a principle everywhere, in one sweep.** If you correct one owner/commit/serialization site,
|
||||
`grep` for the siblings and fix them too — asymmetric fixes are a recurring smell here.
|
||||
- **Honest states.** An error must not render as an empty state; a swallowed failure must not look like
|
||||
"no results". Reads degrade honestly; writes are conditional (`UPDATE … WHERE status='running'`).
|
||||
- **i18n parity.** HU/EN, zero key drift. Every user-facing string is translated.
|
||||
- **Comment the "why", keep it current.** This codebase uses comments as its architecture doc; a stale
|
||||
"why" is a liability, so update it when the behavior changes.
|
||||
|
||||
---
|
||||
|
||||
*A fuller written-conventions doc (`docs/development.md`) is planned; once it lands, this file slims to a
|
||||
pointer plus the gate/workflow/no-attribution rules. Until then, the table above is the source of truth.*
|
||||
+6
-3
@@ -10,7 +10,7 @@ ARG GIT_SHA=unknown
|
||||
ARG BUILD_DATE=
|
||||
|
||||
# Stage 1: build the frontend SPA
|
||||
FROM node:20-alpine AS frontend
|
||||
FROM node:22-alpine AS frontend
|
||||
WORKDIR /fe
|
||||
# Deps first, keyed on the lockfile, so editing app source never re-resolves node_modules.
|
||||
COPY frontend/package.json frontend/package-lock.json ./
|
||||
@@ -57,9 +57,12 @@ RUN apt-get update \
|
||||
&& rm /tmp/deno.zip \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY backend/requirements.txt .
|
||||
# Install from the PINNED lock (requirements.lock, generated from requirements.txt) so the image is
|
||||
# byte-reproducible and the exact versions are the ones pip-audit scanned in CI — not whatever floats
|
||||
# up from the floor ranges on build day. Regenerate the lock when requirements.txt changes (see its header).
|
||||
COPY backend/requirements.lock .
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --upgrade pip && pip install -r requirements.txt
|
||||
pip install --upgrade pip && pip install -r requirements.lock
|
||||
|
||||
COPY --chown=appuser backend/ .
|
||||
# Read at runtime by config.py as the app_version (a committed single source of truth).
|
||||
|
||||
+10
-2
@@ -8,5 +8,13 @@
|
||||
# docker run --rm -v "<repo>/backend:/app" -w /app siftlode-test:local python -m pytest
|
||||
FROM python:3.12-slim
|
||||
WORKDIR /app
|
||||
COPY requirements.txt requirements-dev.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt
|
||||
# ffmpeg so the thumbnail/poster tests (needs_ffmpeg) actually run in the gate instead of skipping
|
||||
# silently — the runtime image already ships ffmpeg, so the test image should too (see conftest's
|
||||
# SIFTLODE_TEST_REQUIRE_DB, which now also refuses to run if ffmpeg is missing).
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ffmpeg \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
# Install the SAME pinned lock as the prod image (+ the test-only tooling), so the tests run against
|
||||
# the exact dependency versions prod ships.
|
||||
COPY requirements.lock requirements-dev.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements.lock -r requirements-dev.txt
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""drop 10 redundant single-column FK indexes (R12 S5)
|
||||
|
||||
Each of these columns already leads a composite UNIQUE constraint, whose btree covers leftmost-prefix
|
||||
lookups on that column (and the ON DELETE CASCADE reverse lookups too) — so the extra single-column
|
||||
`index=True` was pure write-amplification. Dropping them costs nothing on reads and saves an index
|
||||
write per insert/update. Fast (index-only drop, no table rewrite). Reversible (downgrade recreates).
|
||||
|
||||
Redundant index -> the composite that already covers it:
|
||||
ix_subscriptions_user_id <- uq_user_channel (user_id, channel_id)
|
||||
ix_video_states_user_id <- uq_user_video (user_id, video_id)
|
||||
ix_search_finds_user_id <- uq_user_search_video (user_id, video_id)
|
||||
ix_explored_channels_user_id <- uq_user_explored_channel (user_id, channel_id)
|
||||
ix_blocked_channels_user_id <- uq_user_blocked_channel (user_id, channel_id)
|
||||
ix_saved_views_user_id <- uq_user_saved_view (user_id, name)
|
||||
ix_playlist_items_playlist_id <- uq_playlist_video (playlist_id, video_id)
|
||||
ix_download_shares_job_id <- uq_download_share (job_id, to_user_id)
|
||||
ix_plex_states_user_id <- uq_plex_user_item (user_id, item_id)
|
||||
ix_plex_playlist_items_playlist_id <- uq_plex_playlist_item (playlist_id, item_id)
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0062_drop_redundant_idx"
|
||||
down_revision: Union[str, None] = "0061_dl_job_error_code"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
# (index_name, table_name, column)
|
||||
_REDUNDANT: list[tuple[str, str, str]] = [
|
||||
("ix_subscriptions_user_id", "subscriptions", "user_id"),
|
||||
("ix_video_states_user_id", "video_states", "user_id"),
|
||||
("ix_search_finds_user_id", "search_finds", "user_id"),
|
||||
("ix_explored_channels_user_id", "explored_channels", "user_id"),
|
||||
("ix_blocked_channels_user_id", "blocked_channels", "user_id"),
|
||||
("ix_saved_views_user_id", "saved_views", "user_id"),
|
||||
("ix_playlist_items_playlist_id", "playlist_items", "playlist_id"),
|
||||
("ix_download_shares_job_id", "download_shares", "job_id"),
|
||||
("ix_plex_states_user_id", "plex_states", "user_id"),
|
||||
("ix_plex_playlist_items_playlist_id", "plex_playlist_items", "playlist_id"),
|
||||
]
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for idx, _table, _col in _REDUNDANT:
|
||||
# IF EXISTS: a DB whose schema was stamped from a create_all (tests) won't have these once
|
||||
# the models drop index=True, so a plain drop_index would error there.
|
||||
op.execute(f"DROP INDEX IF EXISTS {idx}")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for idx, table, col in _REDUNDANT:
|
||||
op.create_index(idx, table, [col])
|
||||
@@ -0,0 +1,39 @@
|
||||
"""plex library HU-overlay columns (E5 S5a)
|
||||
|
||||
Adds `metadata_lang` (default 'en') and a self-referential `overlay_of` to plex_libraries. An
|
||||
OVERLAY library is the Hungarian-metadata twin of another section: it is never browsed on its own —
|
||||
its titles' HU fields are written onto the matched EN rows (paired by `guid`, see 0064). Cheap:
|
||||
metadata-only, no table rewrite.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0063_plex_library_overlay"
|
||||
down_revision: Union[str, None] = "0062_drop_redundant_idx"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"plex_libraries",
|
||||
sa.Column("metadata_lang", sa.String(8), nullable=False, server_default="en"),
|
||||
)
|
||||
op.add_column(
|
||||
"plex_libraries",
|
||||
sa.Column(
|
||||
"overlay_of",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("plex_libraries.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
),
|
||||
)
|
||||
op.create_index("ix_plex_libraries_overlay_of", "plex_libraries", ["overlay_of"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_plex_libraries_overlay_of", table_name="plex_libraries")
|
||||
op.drop_column("plex_libraries", "overlay_of")
|
||||
op.drop_column("plex_libraries", "metadata_lang")
|
||||
@@ -0,0 +1,58 @@
|
||||
"""plex HU-overlay item/show columns + guid (E5 S5a)
|
||||
|
||||
On plex_items and plex_shows: `guid` (canonical external id, the EN<->HU pairing key), the Hungarian
|
||||
twin fields `title_hu`/`summary_hu`/`genres_hu`, and a STORED generated `search_vector_hu` mirroring
|
||||
the EN vector (title_hu A / people_text B / left(summary_hu,1000) C) + a GIN index. people_text is
|
||||
language-neutral, reused for both vectors.
|
||||
|
||||
Run BEFORE the first HU overlay sync: while the *_hu columns are all NULL the generated-column build
|
||||
is trivial. ADD only (no drop/rebuild of the EN search_vector). NOTE: adding a STORED generated
|
||||
column + its GIN takes an ACCESS EXCLUSIVE lock and rewrites the table — brief, but flag it on prod.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0064_plex_hu_overlay_cols"
|
||||
down_revision: Union[str, None] = "0063_plex_library_overlay"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
_TABLES = ("plex_items", "plex_shows")
|
||||
|
||||
# Mirror of the EN search_vector (models.py) but over the HU twin columns; people_text is shared.
|
||||
_VECTOR_HU = (
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(title_hu, '')), 'A') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(people_text, '')), 'B') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(summary_hu, ''), 1000)), 'C')"
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for t in _TABLES:
|
||||
op.add_column(t, sa.Column("guid", sa.String(64), nullable=True))
|
||||
op.add_column(t, sa.Column("title_hu", sa.String(512), nullable=True))
|
||||
op.add_column(t, sa.Column("summary_hu", sa.Text(), nullable=True))
|
||||
op.add_column(t, sa.Column("genres_hu", postgresql.JSONB(), nullable=True))
|
||||
op.create_index(f"ix_{t}_guid", t, ["guid"])
|
||||
op.create_index(f"ix_{t}_genres_hu_gin", t, ["genres_hu"], postgresql_using="gin")
|
||||
# Generated column can't be expressed via add_column — raw DDL, matching the EN idiom (0044).
|
||||
op.execute(
|
||||
f"ALTER TABLE {t} ADD COLUMN search_vector_hu tsvector "
|
||||
f"GENERATED ALWAYS AS ({_VECTOR_HU}) STORED"
|
||||
)
|
||||
op.execute(f"CREATE INDEX ix_{t}_search_vector_hu ON {t} USING gin (search_vector_hu)")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for t in _TABLES:
|
||||
op.execute(f"DROP INDEX IF EXISTS ix_{t}_search_vector_hu")
|
||||
op.execute(f"ALTER TABLE {t} DROP COLUMN IF EXISTS search_vector_hu")
|
||||
op.drop_index(f"ix_{t}_genres_hu_gin", table_name=t)
|
||||
op.drop_index(f"ix_{t}_guid", table_name=t)
|
||||
op.drop_column(t, "genres_hu")
|
||||
op.drop_column(t, "summary_hu")
|
||||
op.drop_column(t, "title_hu")
|
||||
op.drop_column(t, "guid")
|
||||
@@ -0,0 +1,45 @@
|
||||
"""plex rich-detail item/show columns (E5 S5b)
|
||||
|
||||
On plex_items and plex_shows: the fields that were previously read LIVE from plex.metadata() on every
|
||||
info-page view — `imdb_rating` (the IMDb-specific score, distinct from `rating` = audienceRating),
|
||||
`imdb_id` (tt…), `tagline`, and `full_cast` (ordered [{name, role, thumb}]). Persisting them lets the
|
||||
detail endpoints serve from the DB row instead of a live Plex call (outage-resilient, and the
|
||||
precondition for S6's pure-DB language-aware serving).
|
||||
|
||||
These are filled by the same rich backfill that already fills the full cast (`_enrich_people_batch`),
|
||||
so the data step below resets `people_enriched_at` to NULL on movie/show rows: the backfill re-fetches
|
||||
each once (60/run, self-throttled) and stamps the new columns. ADD-only, all nullable → no table
|
||||
rewrite, no generated column, no ACCESS EXCLUSIVE lock (unlike 0064).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0065_plex_detail_fields"
|
||||
down_revision: Union[str, None] = "0064_plex_hu_overlay_cols"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
_TABLES = ("plex_items", "plex_shows")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
for t in _TABLES:
|
||||
op.add_column(t, sa.Column("imdb_rating", sa.Float(), nullable=True))
|
||||
op.add_column(t, sa.Column("imdb_id", sa.String(16), nullable=True))
|
||||
op.add_column(t, sa.Column("tagline", sa.Text(), nullable=True))
|
||||
op.add_column(t, sa.Column("full_cast", postgresql.JSONB(), nullable=True))
|
||||
# Force a one-time re-enrich so already-enriched rows gain the new columns (episodes are never
|
||||
# enriched, so their NULL people_enriched_at is unaffected; they keep serving detail live).
|
||||
op.execute("UPDATE plex_items SET people_enriched_at = NULL WHERE kind = 'movie'")
|
||||
op.execute("UPDATE plex_shows SET people_enriched_at = NULL")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for t in _TABLES:
|
||||
op.drop_column(t, "full_cast")
|
||||
op.drop_column(t, "tagline")
|
||||
op.drop_column(t, "imdb_id")
|
||||
op.drop_column(t, "imdb_rating")
|
||||
@@ -0,0 +1,24 @@
|
||||
"""notifications.created_at NOT NULL (align schema to the model)
|
||||
|
||||
The model has always declared `Notification.created_at: Mapped[datetime]` (NOT NULL) with
|
||||
`server_default=func.now()`, but migration 0017 created the column without `nullable=False`, so the live
|
||||
schema allowed NULL — a model/schema drift the new migration drift-test (E1 S2) surfaced. The
|
||||
server_default has filled every row since day one, so no NULLs exist; enforcing NOT NULL just makes the
|
||||
schema match the contract the model already promises. Pure DDL, reversible.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0066_notif_created_at_notnull"
|
||||
down_revision: Union[str, None] = "0065_plex_detail_fields"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column("notifications", "created_at", nullable=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.alter_column("notifications", "created_at", nullable=True)
|
||||
+4
-3
@@ -11,7 +11,7 @@ from app import sysconfig
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.models import Invite, User
|
||||
from app.security import decrypt
|
||||
from app.security import decrypt_optional
|
||||
|
||||
PASSWORD_MIN_LEN = 10
|
||||
|
||||
@@ -122,9 +122,10 @@ def purge_user(db: Session, user: User, background: BackgroundTasks) -> None:
|
||||
user_id = user.id
|
||||
tok = user.token
|
||||
# Both are stored encrypted — decrypt for the revoke call (the access-token fallback was passing
|
||||
# ciphertext, so a token row with no refresh token silently failed to revoke).
|
||||
# ciphertext, so a token row with no refresh token silently failed to revoke). decrypt_optional so a
|
||||
# TOKEN_ENCRYPTION_KEY mismatch can't 500 the purge — a best-effort revoke degrades to "skip". (B-core-2)
|
||||
google_token = (
|
||||
(decrypt(tok.refresh_token_enc) or decrypt(tok.access_token)) if tok else None
|
||||
(decrypt_optional(tok.refresh_token_enc) or decrypt_optional(tok.access_token)) if tok else None
|
||||
)
|
||||
# Raw delete so Postgres applies ON DELETE CASCADE / SET NULL on every dependent table.
|
||||
db.execute(delete(User).where(User.id == user_id))
|
||||
|
||||
@@ -221,6 +221,10 @@ class Settings(BaseSettings):
|
||||
plex_path_map: str = ""
|
||||
# Which Plex library section keys are exposed (comma-separated). Empty = all enabled sections.
|
||||
plex_libraries: str = ""
|
||||
# HU-overlay mapping (E5 S5): comma-separated `huSectionKey:enSectionKey` pairs. Each named HU
|
||||
# section is synced as a non-browsable overlay whose Hungarian metadata is written onto the EN
|
||||
# twin section's rows (matched by guid). Empty = no overlays. E.g. "3:4,5:1".
|
||||
plex_overlay_map: str = ""
|
||||
# Stable client identifier Siftlode presents to Plex (X-Plex-Client-Identifier).
|
||||
plex_client_id: str = "siftlode-server"
|
||||
# How often the catalog sync job mirrors Plex metadata, in minutes.
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Per-source adapters — a thin Strategy over yt-dlp, keyed by the media source (youtube / facebook
|
||||
/ reddit / generic).
|
||||
|
||||
yt-dlp stays the download engine; an adapter only contributes source-specific POLICY around it. The
|
||||
base class IS the generic behaviour (yt-dlp does everything, the ref is already a URL), and a
|
||||
subclass overrides ONLY what its source genuinely needs. This replaces knowledge that used to be
|
||||
scattered across worker.py / backfill.py / service.py with no shared source identity.
|
||||
|
||||
Adapters are stateless singletons; get one with `resolve_adapter(source_kind, source_ref, info)`.
|
||||
Resolution works both before extraction (from source_kind / URL host) and after (from the yt-dlp
|
||||
`extractor_key` in the info dict), so a non-YouTube "url" job still lands on the right adapter.
|
||||
"""
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
class SourceAdapter:
|
||||
"""Generic source: yt-dlp handles everything and the ref is already a fetchable URL."""
|
||||
|
||||
key = "generic"
|
||||
|
||||
# --- extraction (worker._extract_with_fallback / backfill) ---------------------------------
|
||||
def client_sets(self) -> list:
|
||||
"""Player-client sets to attempt, in order. A generic source needs no client override, so a
|
||||
single default attempt (`[None]`) — the per-client fallback is a YouTube-only concept."""
|
||||
return [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
"""yt-dlp options this source contributes (merged into `build_ydl_opts`). `clients` is the
|
||||
current entry from `client_sets()`. Generic: nothing."""
|
||||
return {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
"""Retry the extraction with the NEXT client set on this error? Generic: no (single attempt)."""
|
||||
return False
|
||||
|
||||
# --- metadata -----------------------------------------------------------------------------
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
"""The channel/uploader page URL. Generic: whatever yt-dlp exposes directly (YouTube gives
|
||||
`channel_url`), else None (reddit / a raw media URL have no derivable channel page)."""
|
||||
return info.get("channel_url") or info.get("uploader_url")
|
||||
|
||||
def wants_metadata_backfill(self) -> bool:
|
||||
"""Whether the one-off metadata backfill should re-fetch this source. Generic: no."""
|
||||
return False
|
||||
|
||||
# --- source ref ↔ URL ---------------------------------------------------------------------
|
||||
def source_url(self, source_ref: str) -> str:
|
||||
"""A fetchable URL for `source_ref`. Generic: the ref is already a URL."""
|
||||
return source_ref
|
||||
|
||||
def nfo_uniqueid_type(self) -> str:
|
||||
"""The `<uniqueid type=...>` written into the Kodi/Plex `.nfo` sidecar."""
|
||||
return self.key
|
||||
|
||||
|
||||
class YouTubeAdapter(SourceAdapter):
|
||||
key = "youtube"
|
||||
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
||||
_RETRIABLE = (
|
||||
"not a bot",
|
||||
"DRM protected",
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
)
|
||||
|
||||
def client_sets(self) -> list:
|
||||
return [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
|
||||
def ydl_opts_extra(self, clients) -> dict:
|
||||
# == --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…" (list values).
|
||||
ea: dict = {}
|
||||
if clients:
|
||||
ea["youtube"] = {"player_client": clients}
|
||||
if settings.download_pot_base_url:
|
||||
ea["youtubepot-bgutilhttp"] = {"base_url": [settings.download_pot_base_url]}
|
||||
return {"extractor_args": ea} if ea else {}
|
||||
|
||||
def retriable(self, error_text: str) -> bool:
|
||||
return any(m in error_text for m in self._RETRIABLE)
|
||||
|
||||
def wants_metadata_backfill(self) -> bool:
|
||||
return True
|
||||
|
||||
def source_url(self, source_ref: str) -> str:
|
||||
return f"https://www.youtube.com/watch?v={source_ref}"
|
||||
|
||||
|
||||
class FacebookAdapter(SourceAdapter):
|
||||
key = "facebook"
|
||||
|
||||
def uploader_url(self, info: dict) -> str | None:
|
||||
# Facebook gives neither channel_url nor uploader_url, but exposes a numeric `uploader_id`
|
||||
# (the page id) that resolves at facebook.com/<id>.
|
||||
base = super().uploader_url(info)
|
||||
if base:
|
||||
return base
|
||||
uid = info.get("uploader_id")
|
||||
return f"https://www.facebook.com/{uid}" if uid else None
|
||||
|
||||
def wants_metadata_backfill(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
class RedditAdapter(SourceAdapter):
|
||||
key = "reddit"
|
||||
|
||||
|
||||
_YOUTUBE = YouTubeAdapter()
|
||||
_FACEBOOK = FacebookAdapter()
|
||||
_REDDIT = RedditAdapter()
|
||||
_GENERIC = SourceAdapter()
|
||||
|
||||
|
||||
def _host_is(host: str, *domains: str) -> bool:
|
||||
"""True if `host` is exactly one of `domains` or a subdomain of it — a netloc-scoped match, NOT a
|
||||
loose substring, so a look-alike host ('myyoutube.com', 'youtube.com.evil') never matches."""
|
||||
return any(host == d or host.endswith("." + d) for d in domains)
|
||||
|
||||
|
||||
def resolve_adapter(
|
||||
source_kind: str | None, source_ref: str | None = None, info: dict | None = None
|
||||
) -> SourceAdapter:
|
||||
"""Pick the adapter for a job. Matches on `source_kind`, the yt-dlp `extractor_key` from `info`
|
||||
(post-extraction, authoritative), and the netloc of any URL we hold — the `source_ref`
|
||||
(pre-extraction) and yt-dlp's `webpage_url` (post-extraction). Host matching is netloc-scoped, and
|
||||
the webpage_url is used by HOST only (never a substring over its path, which could carry a sibling
|
||||
source's name). Falls back to the generic adapter — behaviour unchanged for any un-dedicated source."""
|
||||
extractor = (info.get("extractor_key") or info.get("extractor") or "").lower() if info else ""
|
||||
hosts = []
|
||||
if source_ref:
|
||||
hosts.append(urlparse(source_ref).netloc.lower())
|
||||
if info and info.get("webpage_url"):
|
||||
hosts.append(urlparse(info["webpage_url"]).netloc.lower())
|
||||
|
||||
def host_is(*domains: str) -> bool:
|
||||
return any(_host_is(h, *domains) for h in hosts)
|
||||
|
||||
if source_kind == "youtube" or "youtube" in extractor or host_is(
|
||||
"youtube.com", "youtu.be", "youtube-nocookie.com"
|
||||
):
|
||||
return _YOUTUBE
|
||||
if "facebook" in extractor or host_is("facebook.com", "fb.watch"):
|
||||
return _FACEBOOK
|
||||
if "reddit" in extractor or host_is("reddit.com"):
|
||||
return _REDDIT
|
||||
return _GENERIC
|
||||
@@ -11,40 +11,26 @@ Run once per environment:
|
||||
docker exec siftlode-worker-1 python -c "from app.downloads import backfill; print(backfill.run())"
|
||||
"""
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import yt_dlp
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import adapters
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, service
|
||||
from app.models import MediaAsset
|
||||
from app.worker import build_uploader_url
|
||||
|
||||
log = logging.getLogger("siftlode.backfill")
|
||||
|
||||
|
||||
def _should_refetch(asset: MediaAsset) -> bool:
|
||||
"""Only YouTube + Facebook sources have a derivable channel URL worth a network round-trip."""
|
||||
if asset.source_kind == "youtube":
|
||||
return True
|
||||
if asset.source_kind == "url":
|
||||
host = urlparse(asset.source_ref).netloc.lower()
|
||||
return "facebook.com" in host or "fb.watch" in host
|
||||
return False
|
||||
|
||||
|
||||
def _extract_meta(url: str) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, trying the same client sets as the worker (YouTube needs the
|
||||
POT/player-client strategy even just to extract)."""
|
||||
client_sets = [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
for clients in client_sets:
|
||||
def _extract_meta(url: str, adapter: adapters.SourceAdapter) -> dict | None:
|
||||
"""Metadata-only yt-dlp extraction, using the source adapter's client sets + options (YouTube
|
||||
needs the POT/player-client strategy even just to extract)."""
|
||||
for clients in adapter.client_sets():
|
||||
opts = formats.build_ydl_opts(
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, settings.download_pot_base_url, clients
|
||||
{}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients)
|
||||
)
|
||||
opts["skip_download"] = True
|
||||
try:
|
||||
@@ -70,10 +56,11 @@ def run() -> dict:
|
||||
.all()
|
||||
)
|
||||
for a in assets:
|
||||
if a.uploader_url is None and _should_refetch(a):
|
||||
adapter = adapters.resolve_adapter(a.source_kind, a.source_ref)
|
||||
if a.uploader_url is None and adapter.wants_metadata_backfill():
|
||||
url = a.source_webpage_url or service.source_url(a.source_kind, a.source_ref)
|
||||
info = _extract_meta(url)
|
||||
uu = build_uploader_url(info) if info else None
|
||||
info = _extract_meta(url, adapter)
|
||||
uu = adapter.uploader_url(info) if info else None
|
||||
if uu:
|
||||
a.uploader_url = uu
|
||||
filled_url += 1
|
||||
|
||||
@@ -96,13 +96,13 @@ def build_ydl_opts(
|
||||
spec: dict,
|
||||
outtmpl: str,
|
||||
progress_hook,
|
||||
pot_base_url: str | None = None,
|
||||
player_clients: list[str] | None = None,
|
||||
extra: dict | None = None,
|
||||
) -> dict:
|
||||
"""Assemble yt-dlp options for a single download to `outtmpl` (a full path template).
|
||||
|
||||
`pot_base_url` points the bgutil PO-token provider plugin at the sidecar server;
|
||||
`player_clients` picks which YouTube clients to use (POT-capable ones like web use the token)."""
|
||||
`extra` is source-specific options contributed by the SourceAdapter (e.g. YouTube's
|
||||
`extractor_args` with the player-client set + bgutil PO-token base_url); it is merged onto the
|
||||
generic options below. Everything here is source-agnostic."""
|
||||
s = normalize(spec)
|
||||
h = s["max_height"]
|
||||
opts: dict = {
|
||||
@@ -127,16 +127,9 @@ def build_ydl_opts(
|
||||
"socket_timeout": 30,
|
||||
}
|
||||
|
||||
# Extractor args: which YouTube clients to use + where the bgutil PO-token sidecar lives
|
||||
# (== --extractor-args "youtube:player_client=…;youtubepot-bgutilhttp:base_url=…"). Values
|
||||
# are lists in the Python API.
|
||||
ea: dict = {}
|
||||
if player_clients:
|
||||
ea["youtube"] = {"player_client": player_clients}
|
||||
if pot_base_url:
|
||||
ea["youtubepot-bgutilhttp"] = {"base_url": [pot_base_url]}
|
||||
if ea:
|
||||
opts["extractor_args"] = ea
|
||||
# Source-specific options from the adapter (e.g. YouTube's extractor_args: player_client + POT).
|
||||
if extra:
|
||||
opts.update(extra)
|
||||
|
||||
if s["mode"] == "a":
|
||||
opts["format"] = "bestaudio/best"
|
||||
|
||||
@@ -43,10 +43,16 @@ class QuotaExceeded(Exception):
|
||||
def resolve(db: Session, user_id: int) -> QuotaLimits:
|
||||
row = db.get(DownloadQuota, user_id)
|
||||
if row is not None:
|
||||
return QuotaLimits(row.max_bytes, row.max_concurrent, row.max_jobs, row.unlimited)
|
||||
# Floor max_concurrent at 1 (admin_set_quota already does on write; the sysconfig default's
|
||||
# min is 1). A stored 0 is nonsensical for a concurrency limit AND would diverge the two
|
||||
# "at limit" checks — the worker's claim-window exclusion vs at_concurrency_limit — since a
|
||||
# 0-limit user is "capped" with zero running jobs, which the SQL exclusion can't see. Keep
|
||||
# them consistent by never letting the effective limit drop below 1. (To BLOCK a user's
|
||||
# downloads, suspend them or set max_jobs=0, not max_concurrent=0.)
|
||||
return QuotaLimits(row.max_bytes, max(1, row.max_concurrent), row.max_jobs, row.unlimited)
|
||||
return QuotaLimits(
|
||||
max_bytes=sysconfig.get_int(db, "download_default_max_bytes"),
|
||||
max_concurrent=sysconfig.get_int(db, "download_default_max_concurrent"),
|
||||
max_concurrent=max(1, sysconfig.get_int(db, "download_default_max_concurrent")),
|
||||
max_jobs=sysconfig.get_int(db, "download_default_max_jobs"),
|
||||
unlimited=False,
|
||||
)
|
||||
|
||||
@@ -8,13 +8,15 @@ the worker live in `worker.py` / the routes; this module owns enqueue + asset bo
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.config import settings
|
||||
from app.downloads import adapters
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import formats, quota, storage
|
||||
from app.downloads.status import HOLDING, JobStatus
|
||||
from app.models import Channel, DownloadJob, MediaAsset, Video
|
||||
|
||||
|
||||
@@ -27,10 +29,9 @@ class EditError(Exception):
|
||||
|
||||
|
||||
def source_url(source_kind: str, source_ref: str) -> str:
|
||||
"""The URL yt-dlp should fetch. YouTube stores the 11-char id; ad-hoc stores the URL."""
|
||||
if source_kind == "youtube":
|
||||
return f"https://www.youtube.com/watch?v={source_ref}"
|
||||
return source_ref
|
||||
"""The URL yt-dlp should fetch. YouTube stores the 11-char id (the adapter reconstructs the watch
|
||||
URL); ad-hoc sources store the URL verbatim."""
|
||||
return adapters.resolve_adapter(source_kind, source_ref).source_url(source_ref)
|
||||
|
||||
|
||||
def reference_url(job, asset) -> str | None:
|
||||
@@ -223,6 +224,35 @@ def enqueue_edit(
|
||||
return job
|
||||
|
||||
|
||||
def finalize_if_running(db: Session, job_id: int, **fields) -> bool:
|
||||
"""Write terminal `fields` to a job in ONE atomic ``UPDATE ... WHERE status='running'`` and report
|
||||
whether it fired. This is the single owner of the invariant "a finishing worker child must never
|
||||
resurrect a job a route just stopped": the atomic WHERE closes the read-then-write race an
|
||||
``if job.status == 'running'`` guard leaves open, so a concurrent cancel/pause can no longer be
|
||||
overwritten with done/error (which would re-open the double-release → file-delete-under-live-holder
|
||||
class, since `done`/`error` hold the asset but `canceled` does not — see release_asset).
|
||||
|
||||
Runs in the CALLER's session and does NOT commit — the caller commits it together with its asset
|
||||
writes in one transaction. It flushes first so any pending asset write takes its row lock BEFORE
|
||||
this takes the job's, matching the cancel route's asset-then-job order (no deadlock). `fields` are
|
||||
mapped-column kwargs (never user input). Returns True when the row was still running and got
|
||||
updated, False when a route had already moved it out of 'running'."""
|
||||
if not fields:
|
||||
return False
|
||||
db.flush() # asset-then-job lock order: push pending asset writes before we lock the job row
|
||||
# A Core UPDATE with the guard as a WHERE — same atomic "only while still running" write as the
|
||||
# rest of the codebase's 2.0 statements, no hand-built SQL string. synchronize_session=False: this
|
||||
# is a terminal write, so we don't pay to sync the (about-to-be-discarded) ORM identity map. Bound
|
||||
# StrEnum values bind as their plain str literal (StrEnum is a str subclass).
|
||||
result = db.execute(
|
||||
update(DownloadJob)
|
||||
.where(DownloadJob.id == job_id, DownloadJob.status == JobStatus.RUNNING.value)
|
||||
.values(**fields)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
return result.rowcount > 0
|
||||
|
||||
|
||||
def release_asset(db: Session, job: DownloadJob) -> None:
|
||||
"""Drop this job's hold on its asset when it leaves a holding state. Once NO job holds the
|
||||
asset anymore, delete the file + row immediately — a deleted download should free its disk,
|
||||
@@ -238,7 +268,7 @@ def release_asset(db: Session, job: DownloadJob) -> None:
|
||||
+1 it, and deleting it under that would FK-null the new job's asset (a lost update on ref_count).
|
||||
A ready asset is still freed at ref==0 (its file is the point); an orphaned errored row is cheap
|
||||
(no file) and gets reused+reset by the next enqueue, or reclaimed by a later GC pass."""
|
||||
if not job.asset_id or job.status not in ("queued", "running", "paused", "done", "error"):
|
||||
if not job.asset_id or job.status not in HOLDING:
|
||||
return
|
||||
asset = db.get(MediaAsset, job.asset_id)
|
||||
if asset is None:
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Canonical download-job status values — the ONE source of truth for the lifecycle literals.
|
||||
|
||||
A job's status is written from several places: the worker (claim → running, terminal done/error,
|
||||
requeue), the routes (pause/cancel/redownload/delete), the service (enqueue), and the GC. Keeping the
|
||||
literals here is a data-safety measure, not tidiness. A British/American drift once silently broke
|
||||
cancellation: the supervisor compared the live status against ``"cancelled"`` while the cancel route
|
||||
wrote ``"canceled"`` (models.py), so the supervisor's "a route stopped it → kill the child, free the
|
||||
asset" branch was DEAD. A canceled job's child then ran to completion and wrote ``"done"``
|
||||
unconditionally over the route's ``"canceled"``; because ``done`` counts as a holding state but
|
||||
``canceled`` does not (see ``service.release_asset``), a later delete released the asset a SECOND time,
|
||||
dropping ``ref_count`` below the live holder count and deleting a file still in use. Import these names
|
||||
instead of typing the strings.
|
||||
"""
|
||||
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class JobStatus(StrEnum):
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
PAUSED = "paused"
|
||||
DONE = "done"
|
||||
ERROR = "error"
|
||||
CANCELED = "canceled"
|
||||
|
||||
|
||||
# A job "holds" its media asset (the +1 on ``ref_count`` from enqueue) while in any of these states;
|
||||
# ``release_asset`` on a status OUTSIDE this set is a no-op. ``CANCELED`` is deliberately excluded: the
|
||||
# cancel route releases once while the job is still holding, then sets ``CANCELED``, so a later delete
|
||||
# does not double-release. This invariant only holds while a finishing child cannot flip ``CANCELED``
|
||||
# back into the set — hence the worker's terminal writes are conditional on the row still being RUNNING.
|
||||
HOLDING = frozenset(
|
||||
{JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.PAUSED, JobStatus.DONE, JobStatus.ERROR}
|
||||
)
|
||||
|
||||
# States a route sets to STOP a running job. The worker's supervisor must step in (kill the child, free
|
||||
# the asset, leave the route's status intact) when it observes one of these on a job it is running.
|
||||
STOPPED_BY_ROUTE = frozenset({JobStatus.PAUSED, JobStatus.CANCELED})
|
||||
@@ -59,6 +59,10 @@ class MediaMeta:
|
||||
title: str
|
||||
uploader: str
|
||||
upload_date: date | None
|
||||
# The .nfo <uniqueid type=...> — the source's scraper type (youtube / facebook / …), set from
|
||||
# the SourceAdapter. Defaults to the source-neutral "generic" so an un-updated caller is at worst
|
||||
# uninformative, never actively mislabelling a non-YouTube download as "youtube".
|
||||
uniqueid_type: str = "generic"
|
||||
duration_s: int | None = None
|
||||
description: str | None = None
|
||||
thumbnail_url: str | None = None
|
||||
@@ -170,7 +174,7 @@ def _nfo_xml(meta: MediaMeta) -> str:
|
||||
f" <premiered>{aired}</premiered>",
|
||||
f" <year>{year}</year>",
|
||||
f" <plot>{escape(meta.description or '')}</plot>",
|
||||
f" <uniqueid type=\"youtube\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
||||
f" <uniqueid type=\"{escape(meta.uniqueid_type)}\" default=\"true\">{escape(meta.video_id)}</uniqueid>",
|
||||
]
|
||||
if meta.duration_s:
|
||||
parts.append(f" <runtime>{round(meta.duration_s / 60)}</runtime>")
|
||||
@@ -178,16 +182,23 @@ def _nfo_xml(meta: MediaMeta) -> str:
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _sidecar(media: Path, ext: str) -> Path:
|
||||
"""The sibling sidecar path (`.nfo` / `.jpg`) for a media file. Swaps the extension on the FULL
|
||||
media path — NOT via `media.with_suffix("")` first: stripping the real extension leaves a stem
|
||||
whose OWN last dot may be inside the title (e.g. "Sonka Házilag."), which `with_suffix` would
|
||||
then read as the extension and clobber, orphaning the sidecar under a truncated name."""
|
||||
return media.with_suffix(ext)
|
||||
|
||||
|
||||
def write_sidecars(
|
||||
download_root: str, rel: str, meta: MediaMeta, thumb_src: Path | None
|
||||
) -> bool:
|
||||
"""Write `<basename>.nfo` + `<basename>.jpg` next to the media file. Returns True if written."""
|
||||
media = abs_path(download_root, rel)
|
||||
base = media.with_suffix("")
|
||||
try:
|
||||
base.with_suffix(".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
|
||||
_sidecar(media, ".nfo").write_text(_nfo_xml(meta), encoding="utf-8")
|
||||
if thumb_src and thumb_src.exists():
|
||||
shutil.copyfile(str(thumb_src), str(base.with_suffix(".jpg")))
|
||||
shutil.copyfile(str(thumb_src), str(_sidecar(media, ".jpg")))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
@@ -196,8 +207,7 @@ def write_sidecars(
|
||||
def delete_asset_files(download_root: str, rel: str) -> None:
|
||||
"""Remove the media file and its sidecars; prune now-empty parent dirs (best effort)."""
|
||||
media = abs_path(download_root, rel)
|
||||
base = media.with_suffix("")
|
||||
for p in (media, base.with_suffix(".nfo"), base.with_suffix(".jpg")):
|
||||
for p in (media, _sidecar(media, ".nfo"), _sidecar(media, ".jpg")):
|
||||
try:
|
||||
p.unlink()
|
||||
except OSError:
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
"""Feed read-model: the pure query-building and row-serialization layer behind the feed.
|
||||
|
||||
Extracted from `routes/feed.py` so the read-model has one home instead of living inside the
|
||||
router module — the feed route handlers and the playlist-detail query share these helpers (the
|
||||
projection, the filter builder, and keyset paging). Nothing here defines a route or reaches for
|
||||
request state; it's all `(db, user, params) -> query`, and a malformed cursor raises a plain
|
||||
`ValueError` for the route handler to translate (the tsquery sanitizer lives in `app.utils`).
|
||||
"""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
ChannelTag,
|
||||
ExploredChannel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SearchFind,
|
||||
Subscription,
|
||||
Tag,
|
||||
User,
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.utils import iso, to_tsquery_str
|
||||
|
||||
|
||||
def _channel_url(channel_id: str, handle: str | None) -> str:
|
||||
if handle and handle.startswith("@"):
|
||||
return f"https://www.youtube.com/{handle}"
|
||||
return f"https://www.youtube.com/channel/{channel_id}"
|
||||
|
||||
|
||||
def _saved_expr(user: User):
|
||||
"""A correlated EXISTS labelled `saved`: is this video in the user's Watch later list?
|
||||
(Watch later is the built-in playlist that replaced the old per-video 'saved' status.)"""
|
||||
return (
|
||||
select(1)
|
||||
.select_from(PlaylistItem)
|
||||
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
|
||||
.where(
|
||||
Playlist.user_id == user.id,
|
||||
Playlist.kind == "watch_later",
|
||||
PlaylistItem.video_id == Video.id,
|
||||
)
|
||||
.exists()
|
||||
.label("saved")
|
||||
)
|
||||
|
||||
|
||||
def feed_columns(user: User, state) -> list:
|
||||
"""The shared feed-row projection (joined Channel fields + per-user watch state +
|
||||
`saved`), consumed by both the feed query and the playlist-detail query so a new column
|
||||
is added in exactly one place. `state` is the caller's aliased VideoState (outer-joined)."""
|
||||
return [
|
||||
Video.id,
|
||||
Video.title,
|
||||
Video.channel_id,
|
||||
Channel.title.label("channel_title"),
|
||||
Channel.thumbnail_url.label("channel_thumbnail"),
|
||||
Channel.handle.label("channel_handle"),
|
||||
Video.published_at,
|
||||
Video.thumbnail_url,
|
||||
Video.duration_seconds,
|
||||
Video.view_count,
|
||||
Video.is_short,
|
||||
Video.live_status,
|
||||
func.coalesce(state.status, "new").label("status"),
|
||||
func.coalesce(state.position_seconds, 0).label("position_seconds"),
|
||||
_saved_expr(user),
|
||||
]
|
||||
|
||||
|
||||
def serialize(row) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
"title": row.title,
|
||||
"channel_id": row.channel_id,
|
||||
"channel_title": row.channel_title,
|
||||
"channel_thumbnail": row.channel_thumbnail,
|
||||
"channel_url": _channel_url(row.channel_id, row.channel_handle),
|
||||
"published_at": iso(row.published_at),
|
||||
"thumbnail_url": row.thumbnail_url,
|
||||
"duration_seconds": row.duration_seconds,
|
||||
"view_count": row.view_count,
|
||||
"is_short": row.is_short,
|
||||
"live_status": row.live_status,
|
||||
"status": row.status or "new",
|
||||
"position_seconds": row.position_seconds or 0,
|
||||
"saved": bool(getattr(row, "saved", False)),
|
||||
"watch_url": f"https://www.youtube.com/watch?v={row.id}",
|
||||
}
|
||||
|
||||
|
||||
def filtered_query(
|
||||
db: Session,
|
||||
user: User,
|
||||
*,
|
||||
tags: list[int],
|
||||
tag_mode: str,
|
||||
channel_ids: list[str],
|
||||
q: str | None,
|
||||
min_duration: int | None,
|
||||
max_duration: int | None,
|
||||
max_age_days: int | None,
|
||||
published_after: date | None,
|
||||
published_before: date | None,
|
||||
show_normal: bool,
|
||||
include_shorts: bool,
|
||||
include_live: bool,
|
||||
show: str,
|
||||
scope: str = "my",
|
||||
library_source: str = "organic",
|
||||
exclude_tag_category: str | None = None,
|
||||
) -> tuple[Select, object]:
|
||||
"""Build the feed query (joins + all WHERE filters), shared by /feed and /feed/count.
|
||||
Returns the column-bearing select plus the priority/relevance rank expression for sorting.
|
||||
|
||||
`scope="my"` (default) restricts the feed to the user's own non-hidden subscriptions.
|
||||
`scope="all"` shows every video in the shared catalog (any user's ingested channels);
|
||||
the subscription is then only LEFT-joined so per-channel priority still resolves for
|
||||
channels the user happens to be subscribed to. Per-user watch state stays private in
|
||||
either mode via the VideoState outer join."""
|
||||
state = aliased(VideoState)
|
||||
status_expr = func.coalesce(state.status, "new")
|
||||
position_expr = func.coalesce(state.position_seconds, 0)
|
||||
|
||||
query = select(*feed_columns(user, state)).join(
|
||||
Channel, Channel.id == Video.channel_id
|
||||
)
|
||||
|
||||
if scope == "all":
|
||||
# Whole shared catalog; subscription is optional (only for priority sort).
|
||||
query = query.outerjoin(
|
||||
Subscription,
|
||||
and_(
|
||||
Subscription.channel_id == Video.channel_id,
|
||||
Subscription.user_id == user.id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
# "Mine": the user's own videos = their non-hidden subscriptions OR videos they
|
||||
# surfaced via their own live YouTube search. Both are LEFT-joined so the Source
|
||||
# filter below can include either or both, and the subscription row stays available
|
||||
# for the priority sort.
|
||||
query = query.outerjoin(
|
||||
Subscription,
|
||||
and_(
|
||||
Subscription.channel_id == Video.channel_id,
|
||||
Subscription.user_id == user.id,
|
||||
Subscription.hidden.is_(False),
|
||||
),
|
||||
).outerjoin(
|
||||
SearchFind,
|
||||
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
|
||||
)
|
||||
|
||||
query = query.outerjoin(
|
||||
state, and_(state.video_id == Video.id, state.user_id == user.id)
|
||||
)
|
||||
|
||||
# Hide videos the maintenance job has flagged as currently unplayable (deleted/private/
|
||||
# abandoned). They're either deleted after a grace period or unhidden if they recover;
|
||||
# either way they shouldn't show in any feed view meanwhile.
|
||||
query = query.where(Video.unavailable_since.is_(None))
|
||||
|
||||
# Explored videos are PER-EXPLORER private: a via_explore video (ingested when someone
|
||||
# opened an un-subscribed channel's page) is visible only to users who explored or
|
||||
# subscribe to its channel, so one user's curiosity never leaks into everyone else's
|
||||
# catalog. Applied in BOTH scopes; the channel page reaches its ephemeral videos by
|
||||
# asking for scope=all + channel_ids (the channel is in this accessible set).
|
||||
accessible_explore = (
|
||||
select(Subscription.channel_id).where(Subscription.user_id == user.id)
|
||||
).union(
|
||||
select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id)
|
||||
)
|
||||
query = query.where(
|
||||
or_(
|
||||
Video.via_explore.is_(False),
|
||||
Video.channel_id.in_(accessible_explore),
|
||||
)
|
||||
)
|
||||
|
||||
# Per-user channel blocklist: never show this user videos from a channel they blocked
|
||||
# (applies to feed, Library, explore, and the channel page alike).
|
||||
query = query.where(
|
||||
Video.channel_id.notin_(
|
||||
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
|
||||
)
|
||||
)
|
||||
|
||||
# Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
|
||||
# (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
|
||||
# "organic" = your subscriptions, "search" = videos you found via your own search,
|
||||
# "all" = both. Default "organic" keeps the Mine feed = subscriptions only.
|
||||
if scope == "all":
|
||||
if library_source == "organic":
|
||||
query = query.where(Video.via_search.is_(False))
|
||||
elif library_source == "search":
|
||||
query = query.where(Video.via_search.is_(True))
|
||||
# "all" → both organic and search-discovered videos
|
||||
else:
|
||||
subscribed = Subscription.channel_id.isnot(None)
|
||||
searched = SearchFind.video_id.isnot(None)
|
||||
if library_source == "search":
|
||||
query = query.where(searched)
|
||||
elif library_source == "all":
|
||||
query = query.where(or_(subscribed, searched))
|
||||
else: # "organic" (default): subscriptions only
|
||||
query = query.where(subscribed)
|
||||
|
||||
if channel_ids:
|
||||
# OR across the picked channels; every other filter still applies on top.
|
||||
query = query.where(Video.channel_id.in_(channel_ids))
|
||||
if min_duration is not None:
|
||||
query = query.where(Video.duration_seconds >= min_duration)
|
||||
if max_duration is not None:
|
||||
query = query.where(Video.duration_seconds <= max_duration)
|
||||
if max_age_days is not None:
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400
|
||||
query = query.where(
|
||||
Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)
|
||||
)
|
||||
if published_after is not None:
|
||||
start = datetime.combine(
|
||||
published_after, datetime.min.time(), tzinfo=timezone.utc
|
||||
)
|
||||
query = query.where(Video.published_at >= start)
|
||||
if published_before is not None:
|
||||
end = datetime.combine(
|
||||
published_before, datetime.min.time(), tzinfo=timezone.utc
|
||||
) + timedelta(days=1)
|
||||
query = query.where(Video.published_at < end)
|
||||
# Full-text relevance search over the weighted search document (title > creator keywords +
|
||||
# the queries that surfaced the video > description; see Video.search_vector). YouTube-like
|
||||
# "fuzzy" matching: word-order-independent, multi-word AND, prefix on the word being typed,
|
||||
# accent-insensitive (unaccent_simple). The channel name still matches as a substring so
|
||||
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort.
|
||||
rank_expr = None
|
||||
if q:
|
||||
ts_str = to_tsquery_str(q)
|
||||
if ts_str:
|
||||
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
|
||||
query = query.where(
|
||||
or_(
|
||||
Video.search_vector.op("@@")(tsq),
|
||||
func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")),
|
||||
)
|
||||
)
|
||||
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
|
||||
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
|
||||
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
|
||||
# published_at then id.
|
||||
rank_expr = cast(func.ts_rank(Video.search_vector, tsq) * 1000000, BigInteger)
|
||||
else:
|
||||
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
|
||||
like = func.unaccent(f"%{q}%")
|
||||
query = query.where(
|
||||
or_(
|
||||
func.unaccent(Video.title).ilike(like),
|
||||
func.unaccent(Channel.title).ilike(like),
|
||||
)
|
||||
)
|
||||
|
||||
if tags:
|
||||
# AND across tag categories (e.g. language AND topic narrows), OR within a
|
||||
# category; the any/all toggle controls multiple topic tags.
|
||||
cat_rows = db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tags))).all()
|
||||
by_category: dict[str, list[int]] = {}
|
||||
for tag_id, category in cat_rows:
|
||||
by_category.setdefault(category, []).append(tag_id)
|
||||
# Facet counting drops the category being counted so its own chips don't zero each
|
||||
# other out (standard drill-down faceting: a category's count ignores its own
|
||||
# selections but still honours the other categories' filters).
|
||||
if exclude_tag_category:
|
||||
by_category.pop(exclude_tag_category, None)
|
||||
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||
for category, ids in by_category.items():
|
||||
if category == "topic" and tag_mode == "and" and len(ids) > 1:
|
||||
sub = (
|
||||
select(ChannelTag.channel_id)
|
||||
.where(ChannelTag.tag_id.in_(ids), visible)
|
||||
.group_by(ChannelTag.channel_id)
|
||||
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids)))
|
||||
)
|
||||
else:
|
||||
sub = select(ChannelTag.channel_id).where(
|
||||
ChannelTag.tag_id.in_(ids), visible
|
||||
)
|
||||
query = query.where(Video.channel_id.in_(sub))
|
||||
|
||||
# Content type: Normal / Shorts / Live·Upcoming as a union of enabled types.
|
||||
# Explicit Watched/Saved/Hidden views show every type so nothing goes missing.
|
||||
explicit_view = show in ("watched", "hidden")
|
||||
if not explicit_view:
|
||||
type_clauses = []
|
||||
if show_normal:
|
||||
type_clauses.append(
|
||||
and_(Video.is_short.is_(False), Video.live_status.notin_(LIVE_OR_UPCOMING))
|
||||
)
|
||||
if include_shorts:
|
||||
type_clauses.append(Video.is_short.is_(True))
|
||||
if include_live:
|
||||
type_clauses.append(Video.live_status.in_(LIVE_OR_UPCOMING))
|
||||
query = query.where(or_(*type_clauses) if type_clauses else false())
|
||||
|
||||
if show == "unwatched":
|
||||
query = query.where(status_expr.notin_(("watched", "hidden")))
|
||||
elif show == "in_progress":
|
||||
# Started but not finished: a stored resume position, not yet watched/hidden.
|
||||
query = query.where(
|
||||
and_(position_expr > 0, status_expr.notin_(("watched", "hidden")))
|
||||
)
|
||||
elif show == "watched":
|
||||
query = query.where(status_expr == "watched")
|
||||
elif show == "hidden":
|
||||
query = query.where(status_expr == "hidden")
|
||||
else: # all
|
||||
query = query.where(status_expr != "hidden")
|
||||
|
||||
return query, rank_expr
|
||||
|
||||
|
||||
# --- Keyset (cursor) pagination ---------------------------------------------
|
||||
#
|
||||
# Each sort is described as an ordered list of key columns (expression + direction
|
||||
# + whether it can be NULL + a JSON round-trip kind), always with Video.id as the
|
||||
# final, unique tiebreaker. This drives BOTH the ORDER BY and the keyset WHERE so
|
||||
# they can never drift apart — the prerequisite for correct cursor paging. Keyset
|
||||
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
|
||||
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
||||
# (OFFSET would shift and duplicate/skip rows).
|
||||
def sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"}
|
||||
table: dict[str, list[dict]] = {
|
||||
"newest": [pub],
|
||||
"oldest": [{**pub, "asc": True}],
|
||||
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
|
||||
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
|
||||
"duration_desc": [{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}],
|
||||
"duration_asc": [{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}],
|
||||
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
|
||||
"title_desc": [{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}],
|
||||
"subscribers": [{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}],
|
||||
"subscribers_asc": [{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}],
|
||||
# Your per-channel priority (set in the channel manager), newest first within a tier.
|
||||
# coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
|
||||
"priority": [{"expr": func.coalesce(Subscription.priority, 0), "asc": False, "nullable": False, "kind": "num"}, pub],
|
||||
"priority_asc": [{"expr": func.coalesce(Subscription.priority, 0), "asc": True, "nullable": False, "kind": "num"}, pub],
|
||||
# Deterministic per-seed shuffle: the same seed yields the same total order, so
|
||||
# the cursor stays valid across pages of one shuffled run.
|
||||
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
|
||||
}
|
||||
# Relevance (search-result ranking) only exists when there's a query to rank against;
|
||||
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
|
||||
if rank_expr is not None:
|
||||
table["relevance"] = [
|
||||
{"expr": rank_expr, "asc": False, "nullable": False, "kind": "num"},
|
||||
pub,
|
||||
]
|
||||
return table.get(sort) or table["newest"]
|
||||
|
||||
|
||||
def order_by(keys: list[dict]):
|
||||
cols = []
|
||||
for k in keys:
|
||||
c = k["expr"].asc() if k["asc"] else k["expr"].desc()
|
||||
cols.append(c.nulls_last() if k["nullable"] else c)
|
||||
cols.append(Video.id.asc()) # unique final tiebreaker
|
||||
return cols
|
||||
|
||||
|
||||
def keyset_predicate(keys: list[dict], cursor_id: str):
|
||||
"""WHERE clause selecting rows strictly AFTER the cursor in the sort order.
|
||||
|
||||
Lexicographic over the key columns then id, NULL-aware for `nulls_last`: a NULL
|
||||
sorts after every non-NULL value, so a cursor sitting on a non-NULL value must
|
||||
also pull in the NULL group, and equality treats NULL == NULL."""
|
||||
def eq(k):
|
||||
return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"]
|
||||
|
||||
def after(k):
|
||||
if k["value"] is None:
|
||||
return false() # nulls are last; nothing sorts strictly after them
|
||||
cmp = k["expr"] > k["value"] if k["asc"] else k["expr"] < k["value"]
|
||||
return or_(cmp, k["expr"].is_(None)) if k["nullable"] else cmp
|
||||
|
||||
branches = []
|
||||
for j in range(len(keys)):
|
||||
branches.append(and_(*[eq(keys[i]) for i in range(j)], after(keys[j])))
|
||||
branches.append(and_(*[eq(k) for k in keys], Video.id > cursor_id))
|
||||
return or_(*branches)
|
||||
|
||||
|
||||
def encode_cursor(keys: list[dict], row) -> str:
|
||||
vals = []
|
||||
for i, k in enumerate(keys):
|
||||
v = getattr(row, f"_k{i}")
|
||||
vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v)
|
||||
payload = json.dumps({"k": vals, "id": row.id})
|
||||
return base64.urlsafe_b64encode(payload.encode()).decode()
|
||||
|
||||
|
||||
def decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]:
|
||||
try:
|
||||
payload = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
raw = payload["k"]
|
||||
if len(raw) != len(keys):
|
||||
raise ValueError("key arity mismatch")
|
||||
vals = [
|
||||
datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v
|
||||
for k, v in zip(keys, raw)
|
||||
]
|
||||
return vals, payload["id"]
|
||||
except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError) as exc:
|
||||
# Framework-free: the route handler translates this into an HTTP 400 so the
|
||||
# read-model stays web-layer-agnostic (see routes/feed.get_feed).
|
||||
raise ValueError("Invalid feed cursor") from exc
|
||||
@@ -160,6 +160,32 @@ async def static_cache_headers(request, call_next):
|
||||
return response
|
||||
|
||||
|
||||
# Baseline security response headers (SEC-1). A minimal, safe set that needs no per-route tuning:
|
||||
# nosniff stops MIME-confusion, the Referrer-Policy keeps the (private) app URL from leaking to
|
||||
# external sites the user follows, `frame-ancestors 'none'` blocks click-jacking by refusing to be
|
||||
# iframed anywhere (we never embed our own pages; we embed OTHERS, which this does not affect), and
|
||||
# Permissions-Policy turns off powerful features the app never uses. `X-Frame-Options: DENY` duplicates
|
||||
# the frame-ancestors intent for legacy engines that never implemented CSP frame-ancestors (belt and
|
||||
# suspenders — modern browsers use the CSP directive and ignore the header). A full Content-Security-Policy
|
||||
# (script/style/connect sources) is deferred — the SPA's inline styles + the YouTube/Plex embeds make
|
||||
# it real work — and HSTS belongs on the TLS-terminating edge (the Caddy front door), not here.
|
||||
_SECURITY_HEADERS = {
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Referrer-Policy": "strict-origin-when-cross-origin",
|
||||
"Content-Security-Policy": "frame-ancestors 'none'",
|
||||
"X-Frame-Options": "DENY",
|
||||
"Permissions-Policy": "camera=(), microphone=(), geolocation=()",
|
||||
}
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def security_headers(request, call_next):
|
||||
response = await call_next(request)
|
||||
for name, value in _SECURITY_HEADERS.items():
|
||||
response.headers.setdefault(name, value)
|
||||
return response
|
||||
|
||||
|
||||
app.include_router(health.router)
|
||||
app.include_router(auth_routes.router)
|
||||
app.include_router(sync.router)
|
||||
|
||||
+65
-10
@@ -219,7 +219,7 @@ class Subscription(Base, TimestampMixin):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("users.id", ondelete="CASCADE")
|
||||
)
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
@@ -388,7 +388,7 @@ class VideoState(Base, UpdatedAtMixin):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("users.id", ondelete="CASCADE")
|
||||
)
|
||||
video_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("videos.id", ondelete="CASCADE"), index=True
|
||||
@@ -418,7 +418,7 @@ class SearchFind(Base, TimestampMixin):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("users.id", ondelete="CASCADE")
|
||||
)
|
||||
video_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("videos.id", ondelete="CASCADE"), index=True
|
||||
@@ -443,7 +443,7 @@ class ExploredChannel(Base, TimestampMixin):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("users.id", ondelete="CASCADE")
|
||||
)
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
@@ -462,7 +462,7 @@ class BlockedChannel(Base, TimestampMixin):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("users.id", ondelete="CASCADE")
|
||||
)
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
@@ -619,7 +619,7 @@ class PlaylistItem(Base):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
playlist_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("playlists.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("playlists.id", ondelete="CASCADE")
|
||||
)
|
||||
video_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("videos.id", ondelete="CASCADE"), index=True
|
||||
@@ -644,7 +644,7 @@ class SavedView(Base, TimestampMixin, UpdatedAtMixin):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("users.id", ondelete="CASCADE")
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(80))
|
||||
filters: Mapped[dict] = mapped_column(JSON)
|
||||
@@ -893,7 +893,7 @@ class DownloadShare(Base, TimestampMixin):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
job_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("download_jobs.id", ondelete="CASCADE")
|
||||
)
|
||||
from_user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
@@ -995,6 +995,13 @@ class PlexLibrary(Base, TimestampMixin, UpdatedAtMixin):
|
||||
kind: Mapped[str] = mapped_column(String(16)) # "movie" | "show"
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True, server_default="true")
|
||||
synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# HU-overlay support (E5 S5). `metadata_lang` tags the section's metadata language. An OVERLAY
|
||||
# library (overlay_of set) is the Hungarian-metadata twin of another section: it is never browsable
|
||||
# on its own — its titles' HU fields are written onto the matched EN rows (paired by `guid`).
|
||||
metadata_lang: Mapped[str] = mapped_column(String(8), default="en", server_default="en")
|
||||
overlay_of: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_libraries.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
|
||||
|
||||
class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
@@ -1007,6 +1014,7 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
Index("ix_plex_shows_genres_gin", "genres", postgresql_using="gin"),
|
||||
Index("ix_plex_shows_directors_gin", "directors", postgresql_using="gin"),
|
||||
Index("ix_plex_shows_cast_gin", "cast_names", postgresql_using="gin"),
|
||||
Index("ix_plex_shows_genres_hu_gin", "genres_hu", postgresql_using="gin"),
|
||||
# ix_plex_shows_collections_gin already created with collections (migration 0047).
|
||||
)
|
||||
|
||||
@@ -1015,6 +1023,9 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
library_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# Canonical external id (e.g. "tmdb:12345") — the key that pairs this EN row with its HU-overlay
|
||||
# twin during sync (E5 S5). Indexed for the guid->row overlay match.
|
||||
guid: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
title: Mapped[str] = mapped_column(String(512))
|
||||
summary: Mapped[str | None] = mapped_column(Text)
|
||||
year: Mapped[int | None] = mapped_column(Integer)
|
||||
@@ -1037,6 +1048,17 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
people_text: Mapped[str | None] = mapped_column(Text) # cast+directors, folded into search_vector
|
||||
# Set once the show's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
||||
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
||||
imdb_rating: Mapped[float | None] = mapped_column(Float) # IMDb-specific score (vs `rating` = audienceRating)
|
||||
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
|
||||
tagline: Mapped[str | None] = mapped_column(Text)
|
||||
full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}]
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched,
|
||||
# so serving falls back to the EN columns. `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
TSVECTOR,
|
||||
Computed(
|
||||
@@ -1046,6 +1068,15 @@ class PlexShow(Base, TimestampMixin, UpdatedAtMixin):
|
||||
persisted=True,
|
||||
),
|
||||
)
|
||||
search_vector_hu: Mapped[object | None] = mapped_column(
|
||||
TSVECTOR,
|
||||
Computed(
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(title_hu, '')), 'A') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(people_text, '')), 'B') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(summary_hu, ''), 1000)), 'C')",
|
||||
persisted=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class PlexSeason(Base, TimestampMixin, UpdatedAtMixin):
|
||||
@@ -1077,6 +1108,7 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
Index("ix_plex_items_directors_gin", "directors", postgresql_using="gin"),
|
||||
Index("ix_plex_items_cast_gin", "cast_names", postgresql_using="gin"),
|
||||
Index("ix_plex_items_collections_gin", "collection_keys", postgresql_using="gin"),
|
||||
Index("ix_plex_items_genres_hu_gin", "genres_hu", postgresql_using="gin"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
@@ -1085,6 +1117,9 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
ForeignKey("plex_libraries.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
kind: Mapped[str] = mapped_column(String(16), index=True) # "movie" | "episode"
|
||||
# Canonical external id (e.g. "tmdb:12345") — pairs a MOVIE row with its HU-overlay twin during
|
||||
# sync (E5 S5); indexed for the guid->row match. Episodes carry it too but it's unused for them.
|
||||
guid: Mapped[str | None] = mapped_column(String(64), index=True)
|
||||
show_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("plex_shows.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
@@ -1130,6 +1165,17 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
people_text: Mapped[str | None] = mapped_column(Text)
|
||||
# Set once the movie's FULL cast has been rich-fetched by the backfill; NULL = still the listing top-3.
|
||||
people_enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# Rich-fetch-only detail fields, persisted by the same backfill that fills the full cast (E5 S5b) so
|
||||
# the info page serves from this row instead of a live plex.metadata() call. NULL until enriched.
|
||||
imdb_rating: Mapped[float | None] = mapped_column(Float) # IMDb-specific score (vs `rating` = audienceRating)
|
||||
imdb_id: Mapped[str | None] = mapped_column(String(16)) # tt… → the clickable IMDb link
|
||||
tagline: Mapped[str | None] = mapped_column(Text)
|
||||
full_cast: Mapped[list | None] = mapped_column(JSONB(none_as_null=True)) # ordered [{name, role, thumb}]
|
||||
# Hungarian-metadata twin fields, written by the overlay sync (E5 S5). NULL = no HU twin matched
|
||||
# (serving falls back to the EN columns). `people_text` is language-neutral, reused for both vectors.
|
||||
title_hu: Mapped[str | None] = mapped_column(String(512))
|
||||
summary_hu: Mapped[str | None] = mapped_column(Text)
|
||||
genres_hu: Mapped[list | None] = mapped_column(JSONB(none_as_null=True))
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
TSVECTOR,
|
||||
Computed(
|
||||
@@ -1139,6 +1185,15 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin):
|
||||
persisted=True,
|
||||
),
|
||||
)
|
||||
search_vector_hu: Mapped[object | None] = mapped_column(
|
||||
TSVECTOR,
|
||||
Computed(
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(title_hu, '')), 'A') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(people_text, '')), 'B') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(summary_hu, ''), 1000)), 'C')",
|
||||
persisted=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class PlexPerson(Base, TimestampMixin, UpdatedAtMixin):
|
||||
@@ -1191,7 +1246,7 @@ class PlexState(Base, UpdatedAtMixin):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("users.id", ondelete="CASCADE")
|
||||
)
|
||||
item_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
|
||||
@@ -1260,7 +1315,7 @@ class PlexPlaylistItem(Base):
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
playlist_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("plex_playlists.id", ondelete="CASCADE"), index=True
|
||||
ForeignKey("plex_playlists.id", ondelete="CASCADE")
|
||||
)
|
||||
item_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("plex_items.id", ondelete="CASCADE"), index=True
|
||||
|
||||
@@ -14,6 +14,14 @@ from app.models import Notification
|
||||
# rows are never pruned (the user hasn't seen them yet).
|
||||
READ_SOFT_CAP = 200
|
||||
|
||||
# Hard limit of the Notification.title column, read straight from the model so the clamp
|
||||
# can never drift from the actual column width (a later migration is the single source of
|
||||
# truth). Titles are only an English fallback — the client renders the translated message
|
||||
# from `data` — so clamping an over-long one (e.g. a FB/IG reel whose "title" is the whole
|
||||
# caption, hundreds of chars) is harmless to the UI and prevents a StringDataRightTruncation
|
||||
# that would abort the caller's whole transaction (the batched download_gc run in particular).
|
||||
TITLE_MAX_LEN = Notification.__table__.c.title.type.length
|
||||
|
||||
|
||||
def create_notification(
|
||||
db: Session,
|
||||
@@ -27,6 +35,8 @@ def create_notification(
|
||||
) -> Notification:
|
||||
"""Insert one notification for a user. Callers that batch many inserts in a single
|
||||
transaction can pass commit=False and commit once themselves."""
|
||||
if title is not None and len(title) > TITLE_MAX_LEN:
|
||||
title = title[: TITLE_MAX_LEN - 1] + "…" # clamp to the column limit, add an ellipsis
|
||||
notif = Notification(
|
||||
user_id=user_id, type=type, title=title, body=body, data=data
|
||||
)
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Plex list/leaf card serializers: the small `model -> dict` projections the library grid,
|
||||
playlists, and detail views render. Extracted from routes/plex/browse.py so the row-shaping
|
||||
lives beside the other Plex read-model modules (detail.py, query.py) rather than inside the
|
||||
router. Pure functions — no DB, no request state."""
|
||||
|
||||
from app.models import PlexItem, PlexShow, PlexState
|
||||
|
||||
|
||||
def movie_card(it: PlexItem, st: PlexState | None) -> dict:
|
||||
return {
|
||||
"id": it.rating_key,
|
||||
"type": "movie",
|
||||
"title": it.title,
|
||||
"year": it.year,
|
||||
"duration_seconds": it.duration_s,
|
||||
"thumb": f"/api/plex/image/{it.rating_key}",
|
||||
"playable": it.playable,
|
||||
"status": st.status if st else "new",
|
||||
"position_seconds": st.position_seconds if st else 0,
|
||||
}
|
||||
|
||||
|
||||
def show_card(sh: PlexShow, status: str = "new") -> dict:
|
||||
return {
|
||||
"id": sh.rating_key,
|
||||
"type": "show",
|
||||
"title": sh.title,
|
||||
"year": sh.year,
|
||||
"thumb": f"/api/plex/image/{sh.rating_key}",
|
||||
"season_count": sh.child_count,
|
||||
# Aggregate watch-state across the show's episodes (new | in_progress | watched) → grid badge.
|
||||
"status": status,
|
||||
}
|
||||
|
||||
|
||||
def episode_card(e: PlexItem, st: PlexState | None) -> dict:
|
||||
return {
|
||||
"id": e.rating_key,
|
||||
"type": "episode",
|
||||
"title": e.title,
|
||||
"summary": e.summary,
|
||||
"season_number": e.season_number,
|
||||
"episode_number": e.episode_number,
|
||||
"duration_seconds": e.duration_s,
|
||||
"thumb": f"/api/plex/image/{e.rating_key}",
|
||||
"playable": e.playable,
|
||||
"status": st.status if st else "new",
|
||||
"position_seconds": st.position_seconds if st else 0,
|
||||
}
|
||||
|
||||
|
||||
def leaf_card(
|
||||
it: PlexItem,
|
||||
st: PlexState | None,
|
||||
show_title: str | None = None,
|
||||
show_rk: str | None = None,
|
||||
) -> dict:
|
||||
"""A playable-leaf card (movie or episode). Used by playlists, which can mix both kinds.
|
||||
`show_rk` (the show's rating_key) lets the client group a playlist's episodes by show/season."""
|
||||
if it.kind == "episode":
|
||||
card = episode_card(it, st)
|
||||
card["show_title"] = show_title
|
||||
card["show_id"] = show_rk
|
||||
return card
|
||||
return movie_card(it, st)
|
||||
@@ -85,6 +85,10 @@ class PlexClient:
|
||||
f"/library/sections/{key}/all",
|
||||
params={
|
||||
"type": item_type,
|
||||
# includeGuids=1 carries each item's Guid array (tmdb/imdb/tvdb ids) in the cheap
|
||||
# listing, so the sync can capture the canonical guid without a per-item rich fetch —
|
||||
# the pairing key for the HU overlay (E5 S5).
|
||||
"includeGuids": 1,
|
||||
"X-Plex-Container-Start": start,
|
||||
"X-Plex-Container-Size": size,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Parse the "info page" rich metadata out of a full Plex metadata dict, and persist the subset that
|
||||
the detail endpoints serve from the DB row (E5 S5b).
|
||||
|
||||
Shared by the live info-page path (routes.plex.browse) and the background rich re-sync (plex.sync) so
|
||||
the two never diverge — the same one-parser rule that keeps `people.py`'s cast handling consistent.
|
||||
`rich_detail` is the pure parser; `apply_rich_detail` writes the rich-ONLY fields onto a movie/show row
|
||||
(genres/directors/studio/content_rating already land from the cheap listing at `_apply_facet_fields`)."""
|
||||
|
||||
from app.plex import people
|
||||
|
||||
|
||||
def rich_detail(meta: dict) -> dict:
|
||||
"""Pull the info-page extras out of a full Plex metadata dict: an IMDb score + link, content
|
||||
rating, genres, director(s), studio, tagline, and cast (name/role/photo). Best-effort — any
|
||||
missing field is just omitted."""
|
||||
# Rating: IMDb-ONLY — the explicit `imdb://` entry in the Rating array, or NULL. We deliberately do
|
||||
# NOT fall back to audienceRating/rating here: that is a Plex audience score, not an IMDb score, and
|
||||
# labelling it "imdb_rating" mislabels it. The audience score is served separately (item/show detail
|
||||
# expose the `rating` column) so the UI can show it when there's no IMDb score.
|
||||
imdb_rating = None
|
||||
for r in meta.get("Rating") or []:
|
||||
if str(r.get("image") or "").startswith("imdb://"):
|
||||
try:
|
||||
imdb_rating = round(float(r.get("value")), 1)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
break
|
||||
# IMDb id (tt…) from the Guid list → a clickable link.
|
||||
imdb_id = None
|
||||
for g in meta.get("Guid") or []:
|
||||
gid = str(g.get("id") or "")
|
||||
if gid.startswith("imdb://"):
|
||||
imdb_id = gid.split("imdb://", 1)[1]
|
||||
break
|
||||
return {
|
||||
"imdb_rating": imdb_rating,
|
||||
"imdb_id": imdb_id,
|
||||
"imdb_url": f"https://www.imdb.com/title/{imdb_id}/" if imdb_id else None,
|
||||
"content_rating": meta.get("contentRating"),
|
||||
"genres": [g.get("tag") for g in (meta.get("Genre") or []) if g.get("tag")][:8],
|
||||
"directors": [
|
||||
d.get("tag") for d in (meta.get("Director") or []) if d.get("tag")
|
||||
][:4],
|
||||
"studio": meta.get("studio"),
|
||||
"tagline": meta.get("tagline"),
|
||||
"cast": people.cast_from_meta(meta),
|
||||
}
|
||||
|
||||
|
||||
def apply_rich_detail(row, meta: dict) -> None:
|
||||
"""Persist the rich-ONLY detail fields onto a movie/show row so the info page serves them from the
|
||||
DB instead of a live Plex call. Call wherever a rich metadata dict is fetched (the cast backfill +
|
||||
the info-page lazy enrich). Empty cast persists as SQL NULL (none_as_null), so an EN fallback and
|
||||
the `full_cast IS NULL` "not yet enriched" signal stay meaningful."""
|
||||
d = rich_detail(meta)
|
||||
row.imdb_rating = d["imdb_rating"]
|
||||
row.imdb_id = d["imdb_id"]
|
||||
row.tagline = d["tagline"]
|
||||
row.full_cast = d["cast"] or None
|
||||
@@ -13,6 +13,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import PlexCollection, PlexItem, PlexSeason, PlexShow
|
||||
from app.plex.cache import CACHE_DIR
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
_EXT_CT = [("jpg", "image/jpeg"), ("png", "image/png"), ("webp", "image/webp")]
|
||||
_CT_EXT = {ct: ext for ext, ct in _EXT_CT}
|
||||
@@ -24,16 +25,16 @@ SIZES = {"thumb": (400, 600), "art": (1280, 720)}
|
||||
def key_for(db: Session, rating_key: str, variant: str) -> str | None:
|
||||
"""The stored Plex image path for a known rating_key (item/show/season). Only mirrored keys
|
||||
are proxyable — this is not an open image proxy."""
|
||||
it = db.query(PlexItem).filter_by(rating_key=rating_key).first()
|
||||
it = by_rating_key(db, PlexItem, rating_key)
|
||||
if it is not None:
|
||||
return it.art_key if variant == "art" else it.thumb_key
|
||||
sh = db.query(PlexShow).filter_by(rating_key=rating_key).first()
|
||||
sh = by_rating_key(db, PlexShow, rating_key)
|
||||
if sh is not None:
|
||||
return sh.art_key if variant == "art" else sh.thumb_key
|
||||
se = db.query(PlexSeason).filter_by(rating_key=rating_key).first()
|
||||
se = by_rating_key(db, PlexSeason, rating_key)
|
||||
if se is not None:
|
||||
return se.thumb_key
|
||||
col = db.query(PlexCollection).filter_by(rating_key=rating_key).first()
|
||||
col = by_rating_key(db, PlexCollection, rating_key)
|
||||
if col is not None:
|
||||
return col.thumb_key
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Small shared Plex row lookups, so the get-by-rating_key pattern lives in one place."""
|
||||
from typing import TypeVar
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
_M = TypeVar("_M")
|
||||
|
||||
|
||||
def by_rating_key(db: Session, model: type[_M], rating_key: str) -> _M | None:
|
||||
"""The mirrored row for a Plex rating_key, or None. `model` is any of the rating_key-keyed Plex
|
||||
tables (PlexItem / PlexShow / PlexSeason / PlexCollection); the return type is that model, so call
|
||||
sites keep their `Model | None` static type."""
|
||||
return db.execute(
|
||||
select(model).where(model.rating_key == str(rating_key))
|
||||
).scalars().first()
|
||||
@@ -5,7 +5,7 @@ re-sync (plex.sync), so the two paths never diverge."""
|
||||
|
||||
from urllib.parse import quote
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy import select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -55,7 +55,10 @@ def enrich_row_people(db: Session, row, cast: list[dict]) -> bool:
|
||||
dirty = True
|
||||
# Headshot cache: upsert name -> proxied thumb (a later fetch fills a prior NULL; names without a
|
||||
# whitelisted photo still get a row so a card can show a placeholder).
|
||||
existing = {p.name: p for p in db.query(PlexPerson).filter(PlexPerson.name.in_(names))}
|
||||
existing = {
|
||||
p.name: p
|
||||
for p in db.execute(select(PlexPerson).where(PlexPerson.name.in_(names))).scalars()
|
||||
}
|
||||
missing: dict[str, str | None] = {}
|
||||
for c in cast:
|
||||
nm, th = c.get("name"), c.get("thumb")
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Plex library query-shaping: the full-text field-scope rechecks and the sidebar metadata
|
||||
filters that /library and /facets share. Extracted from routes/plex/browse.py so the query
|
||||
builder has one home (and so the HU-overlay column selection lands here, not in the router).
|
||||
Pure query fragments — every function returns SQLAlchemy expressions or a params dict."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import and_, func, or_, type_coerce
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
TS_CONFIG = "public.unaccent_simple"
|
||||
|
||||
# --- Language-aware column pickers (E5 S6) ----------------------------------
|
||||
#
|
||||
# The HU-overlay twin (E5 S5) stores Hungarian metadata in `*_hu` columns on the SAME EN row. When
|
||||
# the app language is `hu` we serve those, falling back to the EN column where a row has no HU twin.
|
||||
# Two flavours of picker, on purpose:
|
||||
# * DISPLAY / FILTER (`title_col`/`genres_col`): `coalesce(*_hu, *)` — an EN fallback so a row with
|
||||
# no HU metadata still shows (and filters on) its English value. (The detail endpoints serve the
|
||||
# HU summary directly off the loaded row, so no `summary_col` SQL helper is needed.)
|
||||
# * SEARCH (`vector_col` + `field_recheck`): the raw `*_hu` column coalesced to '' — MIRRORING the
|
||||
# generated `search_vector_hu` definition (which uses `coalesce(title_hu, '')`, NOT an EN
|
||||
# fallback), so the field recheck stays a strict subset of the GIN vector prefilter (the S2
|
||||
# invariant). `people_text` is language-neutral and shared by both vectors.
|
||||
|
||||
|
||||
def title_col(model, lang: str):
|
||||
return func.coalesce(model.title_hu, model.title) if lang == "hu" else model.title
|
||||
|
||||
|
||||
def genres_col(model, lang: str):
|
||||
# type_coerce so the JSONB comparator (`.contains` / `@>`, GIN-backed) survives the coalesce.
|
||||
if lang == "hu":
|
||||
return type_coerce(func.coalesce(model.genres_hu, model.genres), JSONB)
|
||||
return model.genres
|
||||
|
||||
|
||||
def vector_col(model, lang: str):
|
||||
# RANKING vector. In HU we rank by the Hungarian vector; a row matched only via its English vector
|
||||
# (see `vector_match`) gets rank 0 and sorts last, which is the intended HU-first ordering.
|
||||
return model.search_vector_hu if lang == "hu" else model.search_vector
|
||||
|
||||
|
||||
def vector_match(model, lang: str, tsq):
|
||||
"""The full-text WHERE clause. In HU, match EITHER vector so a row with no HU twin stays findable
|
||||
by its English title (search_vector_hu has no title tokens for such a row); EN matches only the EN
|
||||
vector. Ranking still uses `vector_col` (the HU vector in HU mode)."""
|
||||
if lang == "hu":
|
||||
return or_(
|
||||
model.search_vector_hu.op("@@")(tsq), model.search_vector.op("@@")(tsq)
|
||||
)
|
||||
return model.search_vector.op("@@")(tsq)
|
||||
|
||||
|
||||
# Field-scoped search: the scope-pill in the header search box narrows a full-text query to ONE
|
||||
# facet of the row. `all` uses the GIN-indexed weighted `search_vector` untouched (byte-identical to
|
||||
# the pre-scope behaviour); a specific field keeps that GIN prefilter AND re-checks a per-column
|
||||
# tsvector, so the scoped hits are always a strict subset of Everywhere. The per-field expressions
|
||||
# mirror the `search_vector` Computed definition (same `TS_CONFIG`, same `left(summary, 1000)`
|
||||
# truncation) so scoping never surfaces a match the vector itself wouldn't. `episode` is handled
|
||||
# separately in the query body (it targets episode rows, not the movie/show grid).
|
||||
FIELD_COLUMNS = ("title", "people", "summary", "episode")
|
||||
|
||||
|
||||
def _title_tsv(col):
|
||||
return func.to_tsvector(TS_CONFIG, func.coalesce(col, ""))
|
||||
|
||||
|
||||
def _summary_tsv(col):
|
||||
return func.to_tsvector(TS_CONFIG, func.left(func.coalesce(col, ""), 1000))
|
||||
|
||||
|
||||
def field_recheck(model, field: str, tsq, lang: str = "en"):
|
||||
"""A per-field `to_tsvector(...) @@ tsq` clause for a non-`all`, non-`episode` scope, or None.
|
||||
Each per-field tsvector mirrors the generated `search_vector[_hu]` definition (same TS_CONFIG,
|
||||
same `left(summary, 1000)`). In HU, title/summary re-check BOTH the HU and EN columns so a scoped
|
||||
hit stays a strict subset of `vector_match`'s HU union; `people` is language-neutral."""
|
||||
hu = lang == "hu"
|
||||
if field == "title":
|
||||
cols = [model.title_hu, model.title] if hu else [model.title]
|
||||
return or_(*[_title_tsv(c).op("@@")(tsq) for c in cols])
|
||||
if field == "people":
|
||||
return func.to_tsvector(TS_CONFIG, func.coalesce(model.people_text, "")).op("@@")(tsq)
|
||||
if field == "summary":
|
||||
cols = [model.summary_hu, model.summary] if hu else [model.summary]
|
||||
return or_(*[_summary_tsv(c).op("@@")(tsq) for c in cols])
|
||||
return None
|
||||
|
||||
|
||||
def split_csv(v: str | None) -> list[str]:
|
||||
return [s.strip() for s in v.split(",") if s.strip()] if v else []
|
||||
|
||||
|
||||
_ADDED_WINDOWS = {"24h": 1, "1w": 7, "1m": 30, "6m": 182, "1y": 365}
|
||||
|
||||
|
||||
def added_cutoff(within: str | None) -> datetime | None:
|
||||
days = _ADDED_WINDOWS.get(within or "")
|
||||
return datetime.now(timezone.utc) - timedelta(days=days) if days else None
|
||||
|
||||
|
||||
def meta_filter_conds(model, p: dict, lang: str = "en") -> list:
|
||||
"""The shared metadata-filter conditions (identical column names on plex_items + plex_shows) for
|
||||
`p` (the request's filter params). Duration is movie-only, handled by the caller. Returned as a
|
||||
list so faceted search can apply all-but-one group. DRY for browse, unified, and /facets.
|
||||
Genre chips are language-specific strings, so they filter the active-language genre column."""
|
||||
conds: list = []
|
||||
gsel = split_csv(p.get("genres"))
|
||||
if gsel:
|
||||
gcol = genres_col(model, lang)
|
||||
gc = [gcol.contains([g]) for g in gsel]
|
||||
conds.append(and_(*gc) if p.get("genre_mode") == "all" else or_(*gc))
|
||||
crs = split_csv(p.get("content_ratings"))
|
||||
if crs:
|
||||
conds.append(model.content_rating.in_(crs))
|
||||
if p.get("year_min") is not None:
|
||||
conds.append(model.year >= p["year_min"])
|
||||
if p.get("year_max") is not None:
|
||||
conds.append(model.year <= p["year_max"])
|
||||
if p.get("rating_min") is not None:
|
||||
# Filter by the SAME score the UI shows on the badge (IMDb preferred, audience fallback), so a
|
||||
# ratingMin picked from a displayed IMDb score narrows the right column instead of the raw
|
||||
# audience `rating`. Both are 0–10, so coalesce is scale-consistent.
|
||||
conds.append(func.coalesce(model.imdb_rating, model.rating) >= p["rating_min"])
|
||||
cutoff = added_cutoff(p.get("added_within"))
|
||||
if cutoff is not None:
|
||||
conds.append(model.added_at >= cutoff)
|
||||
for d in split_csv(p.get("directors")): # AND
|
||||
conds.append(model.directors.contains([d]))
|
||||
alist = split_csv(p.get("actors"))
|
||||
if alist: # AND (all) or OR (any) — default OR, mirroring genre_mode
|
||||
ac = [model.cast_names.contains([a]) for a in alist]
|
||||
conds.append(and_(*ac) if p.get("actor_mode") == "all" else or_(*ac))
|
||||
stds = split_csv(p.get("studios"))
|
||||
if stds: # OR
|
||||
conds.append(model.studio.in_(stds))
|
||||
if p.get("collection"):
|
||||
conds.append(model.collection_keys.contains([p["collection"]]))
|
||||
return conds
|
||||
|
||||
|
||||
def apply_meta_filters(q, model, p: dict, lang: str = "en"):
|
||||
conds = meta_filter_conds(model, p, lang)
|
||||
return q.where(*conds) if conds else q
|
||||
|
||||
|
||||
@dataclass
|
||||
class LibraryFilters:
|
||||
"""The shared sidebar filter query-params for /library + /facets, declared once and injected via
|
||||
``Depends()``. `as_dict()` yields the `p` dict the query builders read; `duration_*` is included
|
||||
for /facets' wsq but ignored by `meta_filter_conds` (both callers apply duration separately)."""
|
||||
|
||||
genres: str | None = None
|
||||
genre_mode: str = "any"
|
||||
content_ratings: str | None = None
|
||||
year_min: int | None = None
|
||||
year_max: int | None = None
|
||||
rating_min: float | None = None
|
||||
duration_min: int | None = None
|
||||
duration_max: int | None = None
|
||||
added_within: str | None = None
|
||||
directors: str | None = None
|
||||
actors: str | None = None
|
||||
actor_mode: str = (
|
||||
"any" # how multiple actors combine: any (OR, default) | all (AND)
|
||||
)
|
||||
studios: str | None = None
|
||||
collection: str | None = None
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {
|
||||
"genres": self.genres,
|
||||
"genre_mode": self.genre_mode,
|
||||
"content_ratings": self.content_ratings,
|
||||
"year_min": self.year_min,
|
||||
"year_max": self.year_max,
|
||||
"rating_min": self.rating_min,
|
||||
"duration_min": self.duration_min,
|
||||
"duration_max": self.duration_max,
|
||||
"added_within": self.added_within,
|
||||
"directors": self.directors,
|
||||
"actors": self.actors,
|
||||
"actor_mode": self.actor_mode,
|
||||
"studios": self.studios,
|
||||
"collection": self.collection,
|
||||
}
|
||||
|
||||
|
||||
def like_escape(s: str) -> str:
|
||||
return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
@@ -3,7 +3,7 @@
|
||||
Drives the show/season grid badges and the Resume / Play-from-start / mark-all buttons.
|
||||
"""
|
||||
|
||||
from sqlalchemy import and_, case, func, or_
|
||||
from sqlalchemy import and_, case, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.models import PlexItem, PlexState
|
||||
@@ -47,7 +47,7 @@ def agg_subquery(db: Session, user_id: int):
|
||||
grid by aggregate watch-state and (b) compute each card's badge."""
|
||||
ps = aliased(PlexState)
|
||||
return (
|
||||
db.query(
|
||||
select(
|
||||
PlexItem.show_id.label("show_id"),
|
||||
func.count(PlexItem.id).label("total"),
|
||||
func.count(case((ps.status == "watched", 1))).label("watched"),
|
||||
@@ -64,7 +64,7 @@ def agg_subquery(db: Session, user_id: int):
|
||||
).label("inprog"),
|
||||
)
|
||||
.outerjoin(ps, and_(ps.item_id == PlexItem.id, ps.user_id == user_id))
|
||||
.filter(PlexItem.kind == "episode", PlexItem.show_id.isnot(None))
|
||||
.where(PlexItem.kind == "episode", PlexItem.show_id.isnot(None))
|
||||
.group_by(PlexItem.show_id)
|
||||
)
|
||||
|
||||
@@ -72,5 +72,7 @@ def agg_subquery(db: Session, user_id: int):
|
||||
def state_map(db: Session, user_id: int, show_ids: list[int]) -> dict[int, str]:
|
||||
if not show_ids:
|
||||
return {}
|
||||
rows = agg_subquery(db, user_id).filter(PlexItem.show_id.in_(show_ids)).all()
|
||||
rows = db.execute(
|
||||
agg_subquery(db, user_id).where(PlexItem.show_id.in_(show_ids))
|
||||
).all()
|
||||
return {r.show_id: show_status(r.total, r.watched, r.inprog) for r in rows}
|
||||
|
||||
+205
-29
@@ -12,12 +12,14 @@ Cast + intro/credit markers are fetched lazily on the item-detail/player path (P
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.models import PlexCollection, PlexItem, PlexLibrary, PlexSeason, PlexShow
|
||||
from app.plex import people
|
||||
from app.plex import detail, people
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
@@ -80,6 +82,26 @@ def _rating(meta: dict) -> float | None:
|
||||
return None
|
||||
|
||||
|
||||
# Provider preference for the canonical guid (a stable id shared across a title's language twins).
|
||||
_GUID_PREFERENCE = ("tmdb", "imdb", "tvdb")
|
||||
|
||||
|
||||
def _canonical_guid(meta: dict) -> str | None:
|
||||
"""The canonical external id used to pair an EN title with its HU-overlay twin (E5 S5). Reads the
|
||||
`Guid` array (present only with includeGuids=1 in the listing), preferring tmdb > imdb > tvdb,
|
||||
normalised to '<provider>:<id>' (e.g. 'tmdb:12345'). None when no known provider id is present."""
|
||||
found: dict[str, str] = {}
|
||||
for g in meta.get("Guid") or []:
|
||||
raw = str(g.get("id") or "")
|
||||
for prov in _GUID_PREFERENCE:
|
||||
if raw.startswith(f"{prov}://"):
|
||||
found.setdefault(prov, f"{prov}:{raw.split('://', 1)[1]}")
|
||||
for prov in _GUID_PREFERENCE:
|
||||
if prov in found:
|
||||
return found[prov]
|
||||
return None
|
||||
|
||||
|
||||
def _media_facts(meta: dict) -> dict:
|
||||
media = meta.get("Media") or []
|
||||
if not media:
|
||||
@@ -108,21 +130,48 @@ def _enabled_section_keys(db: Session) -> set[str] | None:
|
||||
return keys or None
|
||||
|
||||
|
||||
def _overlay_map(db: Session) -> dict[str, str]:
|
||||
"""The configured HU-overlay pairing: {huSectionKey: enSectionKey} from `plex_overlay_map`
|
||||
('huKey:enKey,...'). Each HU section is synced as a non-browsable overlay onto its EN twin.
|
||||
Malformed pairs are skipped."""
|
||||
raw = sysconfig.get_str(db, "plex_overlay_map")
|
||||
out: dict[str, str] = {}
|
||||
for pair in raw.split(","):
|
||||
hu, sep, en = pair.partition(":")
|
||||
hu, en = hu.strip(), en.strip()
|
||||
if sep and hu and en:
|
||||
out[hu] = en
|
||||
return out
|
||||
|
||||
|
||||
def sync(db: Session) -> dict:
|
||||
"""Full reconcile of the enabled movie/show sections. Returns a small stats dict."""
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
return {"skipped": "disabled"}
|
||||
stats = {"libraries": 0, "movies": 0, "shows": 0, "seasons": 0, "episodes": 0, "collections": 0}
|
||||
stats = {
|
||||
"libraries": 0, "movies": 0, "shows": 0, "seasons": 0, "episodes": 0,
|
||||
"collections": 0, "overlaid": 0,
|
||||
}
|
||||
wanted = _enabled_section_keys(db)
|
||||
overlay_map = _overlay_map(db) # {huKey: enKey}
|
||||
overlay_keys = set(overlay_map)
|
||||
try:
|
||||
with PlexClient(db) as plex:
|
||||
for s in plex.sections():
|
||||
if s.get("type") not in ("movie", "show"):
|
||||
sections = {
|
||||
str(s.get("key")): s
|
||||
for s in plex.sections()
|
||||
if s.get("type") in ("movie", "show")
|
||||
}
|
||||
libs_by_key: dict[str, PlexLibrary] = {}
|
||||
# Pass 1 — browsable (non-overlay) sections. HU-overlay sections are handled in pass 2 and
|
||||
# are NEVER mirrored as browsable libraries of their own.
|
||||
for key, s in sections.items():
|
||||
if key in overlay_keys:
|
||||
continue
|
||||
key = str(s.get("key"))
|
||||
if wanted is not None and key not in wanted:
|
||||
continue
|
||||
lib = _upsert_library(db, key, s.get("title") or key, s.get("type"))
|
||||
libs_by_key[key] = lib
|
||||
stats["libraries"] += 1
|
||||
if s.get("type") == "movie":
|
||||
_sync_movies(db, plex, lib, stats)
|
||||
@@ -133,6 +182,24 @@ def sync(db: Session) -> dict:
|
||||
# libraries already mirrored (idempotent — the next run reconciles the rest), instead
|
||||
# of the old single end-of-loop commit that a late failure rolled back entirely.
|
||||
db.commit()
|
||||
# Pass 2 — HU overlays. The EN twins are now synced (with guids), so write each overlay
|
||||
# section's Hungarian metadata onto the matched EN rows. Requires the EN section to have been
|
||||
# synced in pass 1 (in the allowlist / all) and to share the overlay's kind.
|
||||
for hu_key, en_key in overlay_map.items():
|
||||
s = sections.get(hu_key)
|
||||
en_lib = libs_by_key.get(en_key)
|
||||
if s is None or en_lib is None or s.get("type") != en_lib.kind:
|
||||
log.warning(
|
||||
"Plex overlay skipped: hu=%s en=%s (missing section/EN lib or kind mismatch)",
|
||||
hu_key, en_key,
|
||||
)
|
||||
continue
|
||||
lib = _upsert_library(db, hu_key, s.get("title") or hu_key, s.get("type"))
|
||||
lib.metadata_lang = "hu"
|
||||
lib.overlay_of = en_lib.id
|
||||
db.flush()
|
||||
_sync_overlay(db, plex, lib, en_lib.id, stats)
|
||||
db.commit()
|
||||
# Rich-fetch a bounded batch of not-yet-enriched titles' FULL cast, so the whole library
|
||||
# becomes filterable over successive runs (the cheap listing above only stored the top ~3).
|
||||
try:
|
||||
@@ -165,20 +232,20 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
|
||||
shared enrich_row_people). Oldest-id first, movies before shows. Stamps people_enriched_at only on
|
||||
a SUCCESSFUL fetch, so a transient per-item failure retries next run rather than being lost.
|
||||
Returns how many items were enriched this run."""
|
||||
movies = (
|
||||
db.query(PlexItem)
|
||||
.filter(PlexItem.kind == "movie", PlexItem.people_enriched_at.is_(None))
|
||||
movies = db.execute(
|
||||
select(PlexItem)
|
||||
.where(PlexItem.kind == "movie", PlexItem.people_enriched_at.is_(None))
|
||||
.order_by(PlexItem.id)
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
).scalars().all()
|
||||
remaining = limit - len(movies)
|
||||
shows = (
|
||||
db.query(PlexShow)
|
||||
.filter(PlexShow.people_enriched_at.is_(None))
|
||||
.order_by(PlexShow.id)
|
||||
.limit(remaining)
|
||||
.all()
|
||||
db.execute(
|
||||
select(PlexShow)
|
||||
.where(PlexShow.people_enriched_at.is_(None))
|
||||
.order_by(PlexShow.id)
|
||||
.limit(remaining)
|
||||
).scalars().all()
|
||||
if remaining > 0
|
||||
else []
|
||||
)
|
||||
@@ -191,6 +258,7 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
|
||||
log.debug("Plex people enrich fetch failed for %s: %s", row.rating_key, e)
|
||||
continue # leave people_enriched_at NULL so it's retried on a later run
|
||||
people.enrich_row_people(db, row, people.cast_from_meta(meta))
|
||||
detail.apply_rich_detail(row, meta) # imdb/tagline/full_cast → served from the row, not live
|
||||
row.people_enriched_at = now
|
||||
n += 1
|
||||
db.commit()
|
||||
@@ -198,7 +266,9 @@ def _enrich_people_batch(db: Session, plex: PlexClient, limit: int = _ENRICH_BAT
|
||||
|
||||
|
||||
def _upsert_library(db: Session, key: str, title: str, kind: str) -> PlexLibrary:
|
||||
lib = db.query(PlexLibrary).filter_by(plex_key=key).first()
|
||||
lib = db.execute(
|
||||
select(PlexLibrary).where(PlexLibrary.plex_key == key)
|
||||
).scalars().first()
|
||||
if lib is None:
|
||||
lib = PlexLibrary(plex_key=key)
|
||||
db.add(lib)
|
||||
@@ -224,6 +294,11 @@ def _paginate(plex: PlexClient, key: str, item_type: int):
|
||||
break
|
||||
|
||||
|
||||
# The section listing exposes at most the top ~20 cast (Plex's own cap); a stored cast longer than this
|
||||
# can only have come from the rich enrich, never the listing.
|
||||
_LISTING_CAST_CAP = 20
|
||||
|
||||
|
||||
def _apply_facet_fields(row, meta: dict) -> None:
|
||||
"""Set the filterable/orderable metadata columns shared by plex_items + plex_shows (identical
|
||||
column names) from the cheap section listing: rating, content_rating, studio, release date,
|
||||
@@ -234,7 +309,20 @@ def _apply_facet_fields(row, meta: dict) -> None:
|
||||
row.originally_available_at = _date(meta.get("originallyAvailableAt"))
|
||||
row.genres = _tags(meta, "Genre")
|
||||
row.directors = _tags(meta, "Director")
|
||||
row.cast_names = _tags(meta, "Role", limit=20)
|
||||
# The listing carries only the top ~20 cast, so re-applying it would DOWNGRADE an already-enriched
|
||||
# full cast (up to 24, grown by the rich backfill/info-page enrich), silently breaking the
|
||||
# person-page cast filter for names past the top ~20. The watermark gate below (`people_enriched_at
|
||||
# is None`) normally protects this, but it is NOT enough on its own: a migration that resets the
|
||||
# watermark re-opens the regression (0065 nulled it, which is exactly how the S5b fix got undone in
|
||||
# prod). So guard STRUCTURALLY too, by LENGTH against the listing cap: a stored cast LONGER than the
|
||||
# cap can only be an enriched fuller list, so keep it (robust to any watermark reset). A stored cast
|
||||
# at or below the cap is itself listing-sourced, so take the fresh listing — which tracks Plex cast
|
||||
# EDITS in EITHER direction (comparing against this listing's length instead would silently keep a
|
||||
# stale longer cast when Plex TRIMS one).
|
||||
if row.people_enriched_at is None:
|
||||
listing_cast = _tags(meta, "Role", limit=_LISTING_CAST_CAP)
|
||||
if len(row.cast_names or []) <= _LISTING_CAST_CAP:
|
||||
row.cast_names = listing_cast
|
||||
people = list(dict.fromkeys((row.directors or []) + (row.cast_names or [])))
|
||||
row.people_text = " ".join(people) or None
|
||||
|
||||
@@ -242,6 +330,9 @@ def _apply_facet_fields(row, meta: dict) -> None:
|
||||
def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
|
||||
row.library_id = lib_id
|
||||
row.kind = kind
|
||||
# EN<->HU pairing key (movies; harmless/unused on episodes). Keep the prior guid if this listing
|
||||
# carries none (a server ignoring includeGuids mustn't wipe the overlay's only match key).
|
||||
row.guid = _canonical_guid(meta) or row.guid
|
||||
row.title = (meta.get("title") or "").strip() or "Untitled"
|
||||
row.summary = meta.get("summary")
|
||||
row.year = meta.get("year")
|
||||
@@ -261,7 +352,12 @@ def _apply_item(row: PlexItem, lib_id: int, kind: str, meta: dict) -> None:
|
||||
|
||||
|
||||
def _sync_movies(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
||||
existing = {r.rating_key: r for r in db.query(PlexItem).filter_by(library_id=lib.id, kind="movie")}
|
||||
existing = {
|
||||
r.rating_key: r
|
||||
for r in db.execute(
|
||||
select(PlexItem).where(PlexItem.library_id == lib.id, PlexItem.kind == "movie")
|
||||
).scalars()
|
||||
}
|
||||
for meta in _paginate(plex, lib.plex_key, _TYPE_MOVIE):
|
||||
rk = str(meta.get("ratingKey") or "")
|
||||
if not rk:
|
||||
@@ -276,7 +372,12 @@ def _sync_movies(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -
|
||||
|
||||
def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) -> None:
|
||||
# 1) shows
|
||||
shows = {r.rating_key: r for r in db.query(PlexShow).filter_by(library_id=lib.id)}
|
||||
shows = {
|
||||
r.rating_key: r
|
||||
for r in db.execute(
|
||||
select(PlexShow).where(PlexShow.library_id == lib.id)
|
||||
).scalars()
|
||||
}
|
||||
for meta in _paginate(plex, lib.plex_key, _TYPE_SHOW):
|
||||
rk = str(meta.get("ratingKey") or "")
|
||||
if not rk:
|
||||
@@ -286,6 +387,7 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
||||
db.add(sh)
|
||||
shows[rk] = sh
|
||||
sh.library_id = lib.id
|
||||
sh.guid = _canonical_guid(meta) or sh.guid # EN<->HU pairing key; keep prior if listing has none
|
||||
sh.title = (meta.get("title") or "").strip() or "Untitled"
|
||||
sh.summary = meta.get("summary")
|
||||
sh.year = meta.get("year")
|
||||
@@ -299,7 +401,12 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
||||
show_id = {rk: sh.id for rk, sh in shows.items()}
|
||||
|
||||
# 2) seasons
|
||||
seasons = {r.rating_key: r for r in db.query(PlexSeason).join(PlexShow).filter(PlexShow.library_id == lib.id)}
|
||||
seasons = {
|
||||
r.rating_key: r
|
||||
for r in db.execute(
|
||||
select(PlexSeason).join(PlexShow).where(PlexShow.library_id == lib.id)
|
||||
).scalars()
|
||||
}
|
||||
for meta in _paginate(plex, lib.plex_key, _TYPE_SEASON):
|
||||
rk = str(meta.get("ratingKey") or "")
|
||||
sid = show_id.get(str(meta.get("parentRatingKey") or ""))
|
||||
@@ -318,7 +425,12 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
||||
season_id = {rk: se.id for rk, se in seasons.items()}
|
||||
|
||||
# 3) episodes (playable leaves)
|
||||
episodes = {r.rating_key: r for r in db.query(PlexItem).filter_by(library_id=lib.id, kind="episode")}
|
||||
episodes = {
|
||||
r.rating_key: r
|
||||
for r in db.execute(
|
||||
select(PlexItem).where(PlexItem.library_id == lib.id, PlexItem.kind == "episode")
|
||||
).scalars()
|
||||
}
|
||||
for meta in _paginate(plex, lib.plex_key, _TYPE_EPISODE):
|
||||
rk = str(meta.get("ratingKey") or "")
|
||||
if not rk:
|
||||
@@ -335,6 +447,59 @@ def _sync_shows(db: Session, plex: PlexClient, lib: PlexLibrary, stats: dict) ->
|
||||
db.flush()
|
||||
|
||||
|
||||
def _sync_overlay(
|
||||
db: Session, plex: PlexClient, lib: PlexLibrary, en_lib_id: int, stats: dict
|
||||
) -> None:
|
||||
"""Write an overlay section's Hungarian metadata onto its EN twin's rows, matched by guid. Creates
|
||||
NO browsable rows — an overlay library is never listed on its own. Only title/summary/genres differ
|
||||
by language; the language-neutral people_text feeds both search vectors.
|
||||
|
||||
This is a full RECONCILE: every guid-bearing EN row's HU fields are cleared first, then the HU
|
||||
section repopulates the matches — so an EN row whose HU twin vanished (or was re-guid'd) falls back
|
||||
to EN instead of keeping stale Hungarian metadata. HU items with no EN match are skipped."""
|
||||
if lib.kind == "movie":
|
||||
rows = db.execute(
|
||||
select(PlexItem).where(
|
||||
PlexItem.library_id == en_lib_id,
|
||||
PlexItem.kind == "movie",
|
||||
PlexItem.guid.is_not(None),
|
||||
)
|
||||
).scalars()
|
||||
item_type = _TYPE_MOVIE
|
||||
else:
|
||||
rows = db.execute(
|
||||
select(PlexShow).where(
|
||||
PlexShow.library_id == en_lib_id, PlexShow.guid.is_not(None)
|
||||
)
|
||||
).scalars()
|
||||
item_type = _TYPE_SHOW
|
||||
# guid -> the EN rows carrying it (a list: two editions of one film can share a guid; both overlay).
|
||||
by_guid: dict[str, list] = {}
|
||||
for r in rows:
|
||||
by_guid.setdefault(r.guid, []).append(r)
|
||||
# Reconcile step: clear HU fields on all guid-bearing rows. Only rows that actually held HU data
|
||||
# become dirty (SQLAlchemy skips a no-net-change UPDATE), so this is cheap; matches are re-set below.
|
||||
for group in by_guid.values():
|
||||
for row in group:
|
||||
row.title_hu = row.summary_hu = row.genres_hu = None
|
||||
seen: set[str] = set() # first HU entry for a guid wins; a dup HU item can't overwrite or double-count
|
||||
for meta in _paginate(plex, lib.plex_key, item_type):
|
||||
g = _canonical_guid(meta)
|
||||
targets = by_guid.get(g) if g else None
|
||||
if not targets or g in seen:
|
||||
continue
|
||||
seen.add(g)
|
||||
title_hu = (meta.get("title") or "").strip() or None
|
||||
summary_hu = meta.get("summary")
|
||||
genres_hu = _tags(meta, "Genre")
|
||||
for row in targets:
|
||||
row.title_hu = title_hu
|
||||
row.summary_hu = summary_hu
|
||||
row.genres_hu = genres_hu
|
||||
stats["overlaid"] = stats.get("overlaid", 0) + 1
|
||||
db.flush()
|
||||
|
||||
|
||||
def _upsert_collection(
|
||||
db: Session, existing: PlexCollection | None, rk: str, lib_id: int, meta: dict
|
||||
) -> PlexCollection:
|
||||
@@ -359,17 +524,21 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
||||
collection's membership on the affected member rows."""
|
||||
meta = plex.metadata(rk) or {}
|
||||
col = _upsert_collection(
|
||||
db, db.query(PlexCollection).filter_by(rating_key=rk).first(), rk, lib.id, meta
|
||||
db, by_rating_key(db, PlexCollection, rk), rk, lib.id, meta
|
||||
)
|
||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||
new_members = {str(c.get("ratingKey")) for c in plex.collection_children(rk) if c.get("ratingKey")}
|
||||
current = db.query(Model).filter(Model.collection_keys.contains([rk])).all()
|
||||
current = db.execute(
|
||||
select(Model).where(Model.collection_keys.contains([rk]))
|
||||
).scalars().all()
|
||||
for row in current: # drop rk from rows that are no longer members
|
||||
if row.rating_key not in new_members:
|
||||
row.collection_keys = [k for k in (row.collection_keys or []) if k != rk] or None
|
||||
to_add = new_members - {r.rating_key for r in current}
|
||||
if to_add:
|
||||
for row in db.query(Model).filter(Model.rating_key.in_(to_add)):
|
||||
for row in db.execute(
|
||||
select(Model).where(Model.rating_key.in_(to_add))
|
||||
).scalars():
|
||||
row.collection_keys = list(dict.fromkeys([*(row.collection_keys or []), rk]))
|
||||
db.flush()
|
||||
return col
|
||||
@@ -378,9 +547,11 @@ def resync_collection(db: Session, plex: PlexClient, lib: PlexLibrary, rk: str)
|
||||
def delete_collection_local(db: Session, lib: PlexLibrary, rk: str) -> None:
|
||||
"""Local cleanup after a collection is deleted from Plex: strip rk from members + drop the row."""
|
||||
Model = PlexItem if lib.kind == "movie" else PlexShow
|
||||
for row in db.query(Model).filter(Model.collection_keys.contains([rk])):
|
||||
for row in db.execute(
|
||||
select(Model).where(Model.collection_keys.contains([rk]))
|
||||
).scalars():
|
||||
row.collection_keys = [k for k in (row.collection_keys or []) if k != rk] or None
|
||||
col = db.query(PlexCollection).filter_by(rating_key=rk).first()
|
||||
col = by_rating_key(db, PlexCollection, rk)
|
||||
if col is not None:
|
||||
db.delete(col)
|
||||
db.flush()
|
||||
@@ -391,7 +562,12 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
||||
written as `collection_keys` on member movies (plex_items) / shows (plex_shows). A full reconcile:
|
||||
collections gone from Plex are pruned, and each member row's keys are rebuilt from scratch."""
|
||||
cols = plex.collections(lib.plex_key)
|
||||
existing = {c.rating_key: c for c in db.query(PlexCollection).filter_by(library_id=lib.id)}
|
||||
existing = {
|
||||
c.rating_key: c
|
||||
for c in db.execute(
|
||||
select(PlexCollection).where(PlexCollection.library_id == lib.id)
|
||||
).scalars()
|
||||
}
|
||||
seen: set[str] = set()
|
||||
membership: dict[str, list[str]] = {} # member rating_key -> [collection rating_key, ...]
|
||||
for meta in cols:
|
||||
@@ -410,8 +586,8 @@ def _sync_collections(db: Session, plex: PlexClient, lib: PlexLibrary, stats: di
|
||||
if rk not in seen:
|
||||
db.delete(col)
|
||||
# Rebuild collection_keys on every member row of this library (clear + set).
|
||||
for row in db.query(PlexItem).filter_by(library_id=lib.id):
|
||||
for row in db.execute(select(PlexItem).where(PlexItem.library_id == lib.id)).scalars():
|
||||
row.collection_keys = membership.get(row.rating_key) or None
|
||||
for row in db.query(PlexShow).filter_by(library_id=lib.id):
|
||||
for row in db.execute(select(PlexShow).where(PlexShow.library_id == lib.id)).scalars():
|
||||
row.collection_keys = membership.get(row.rating_key) or None
|
||||
db.flush()
|
||||
|
||||
@@ -16,7 +16,7 @@ history + onDeck, last-write-wins) build on this and land in later ships.
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import func
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -61,7 +61,10 @@ def _plex_watch_to_state(
|
||||
|
||||
def _rk_to_id(db: Session) -> dict[str, int]:
|
||||
"""The rating_key → PlexItem.id lookup used to resolve a Plex leaf back to its mirrored row."""
|
||||
return {rk: iid for rk, iid in db.query(PlexItem.rating_key, PlexItem.id)}
|
||||
return {
|
||||
rk: iid
|
||||
for rk, iid in db.execute(select(PlexItem.rating_key, PlexItem.id)).all()
|
||||
}
|
||||
|
||||
|
||||
def _scan_plex_states(
|
||||
@@ -162,7 +165,9 @@ def link_for_push(db: Session, user_id: int) -> PlexLink | None:
|
||||
"""The active owner watch-push link for this user, or None when push is off / not the owner.
|
||||
Push is gated on `initial_import_done` so we never push a half-known local state over Plex's
|
||||
authoritative one before the one-time "Plex is master" import has reconciled them."""
|
||||
link = db.query(PlexLink).filter_by(user_id=user_id).first()
|
||||
link = db.execute(
|
||||
select(PlexLink).where(PlexLink.user_id == user_id)
|
||||
).scalars().first()
|
||||
if link and link.uses_admin and link.sync_enabled and link.initial_import_done:
|
||||
return link
|
||||
return None
|
||||
@@ -217,7 +222,11 @@ def push_state_to_plex(
|
||||
# Flag the row synced — but an "unwatched" via status=new deletes the row, so there may be
|
||||
# nothing to flag; that's fine (unscrobble still happened). Skip the flag if the row was
|
||||
# re-edited since we captured `expect_ts` (leave it dirty for its own push / the reconcile).
|
||||
st = db.query(PlexState).filter_by(user_id=user_id, item_id=item_id).first()
|
||||
st = db.execute(
|
||||
select(PlexState).where(
|
||||
PlexState.user_id == user_id, PlexState.item_id == item_id
|
||||
)
|
||||
).scalars().first()
|
||||
if st is not None and _row_matches(st, expect_ts):
|
||||
st.synced_to_plex = True
|
||||
db.commit()
|
||||
@@ -272,9 +281,11 @@ def push_bulk_state_to_plex(user_id: int, changes: list[tuple[int, str, str]]) -
|
||||
# Flag every successfully-pushed row that still exists (an "unwatched" deletes the row, so
|
||||
# only the "watched" changes have a row to flag).
|
||||
if synced_ids:
|
||||
db.query(PlexState).filter(
|
||||
PlexState.user_id == user_id, PlexState.item_id.in_(synced_ids)
|
||||
).update({PlexState.synced_to_plex: True}, synchronize_session=False)
|
||||
db.execute(
|
||||
update(PlexState)
|
||||
.where(PlexState.user_id == user_id, PlexState.item_id.in_(synced_ids))
|
||||
.values(synced_to_plex=True)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
@@ -290,11 +301,13 @@ _LWW_TOLERANCE_S = 90
|
||||
def _active_sync_links(db: Session) -> list[PlexLink]:
|
||||
"""Every link with two-way sync actually running (owner MVP = one row). Iterated uniformly so
|
||||
P5b friend links slot in later without touching this loop."""
|
||||
return (
|
||||
db.query(PlexLink)
|
||||
.filter_by(uses_admin=True, sync_enabled=True, initial_import_done=True)
|
||||
.all()
|
||||
)
|
||||
return db.execute(
|
||||
select(PlexLink).where(
|
||||
PlexLink.uses_admin.is_(True),
|
||||
PlexLink.sync_enabled.is_(True),
|
||||
PlexLink.initial_import_done.is_(True),
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
|
||||
def _resolve_account_id(db: Session, plex: PlexClient, link: PlexLink) -> int:
|
||||
@@ -340,7 +353,11 @@ def _pull_apply(
|
||||
"""Bring one Plex-side positive signal into Siftlode under last-write-wins. Returns
|
||||
"watched"/"in_progress" if it changed the row, else None. Never clears a row (un-watch is the
|
||||
full reconcile's job) — this only pulls in watched/resume."""
|
||||
st = db.query(PlexState).filter_by(user_id=user_id, item_id=item_id).first()
|
||||
st = db.execute(
|
||||
select(PlexState).where(
|
||||
PlexState.user_id == user_id, PlexState.item_id == item_id
|
||||
)
|
||||
).scalars().first()
|
||||
if _same_state(st, status, pos):
|
||||
if st is not None and not st.synced_to_plex:
|
||||
st.synced_to_plex = True # already matches Plex → just settle the flag, no churn
|
||||
@@ -368,17 +385,18 @@ def _repush_dirty(db: Session, plex: PlexClient, link: PlexLink) -> int:
|
||||
# re-push doesn't need an extra per-item plex.metadata() round-trip just to fill set_timeline's
|
||||
# duration argument.
|
||||
info_by_id: dict[int, tuple[str, int | None]] = {
|
||||
iid: (rk, dur) for iid, rk, dur in db.query(PlexItem.id, PlexItem.rating_key, PlexItem.duration_s)
|
||||
iid: (rk, dur)
|
||||
for iid, rk, dur in db.execute(
|
||||
select(PlexItem.id, PlexItem.rating_key, PlexItem.duration_s)
|
||||
).all()
|
||||
}
|
||||
dirty = (
|
||||
db.query(PlexState)
|
||||
.filter(
|
||||
dirty = db.execute(
|
||||
select(PlexState).where(
|
||||
PlexState.user_id == link.user_id,
|
||||
PlexState.synced_to_plex.is_(False),
|
||||
PlexState.status != "hidden",
|
||||
)
|
||||
.all()
|
||||
)
|
||||
).scalars().all()
|
||||
n = 0
|
||||
for st in dirty:
|
||||
info = info_by_id.get(st.item_id)
|
||||
@@ -477,7 +495,9 @@ def run_plex_watch_reconcile(db: Session) -> dict:
|
||||
|
||||
# Siftlode rows Plex no longer has a record for: clear the ones that were mirrored
|
||||
# (Plex un-watched them); leave hidden + dirty (dirty gets re-pushed just below).
|
||||
for st in db.query(PlexState).filter_by(user_id=link.user_id).all():
|
||||
for st in db.execute(
|
||||
select(PlexState).where(PlexState.user_id == link.user_id)
|
||||
).scalars().all():
|
||||
if st.item_id in plex_states or st.status == "hidden":
|
||||
continue
|
||||
if st.synced_to_plex:
|
||||
|
||||
@@ -118,6 +118,20 @@ def can_spend(db: Session, units: int) -> bool:
|
||||
return remaining_today(db) >= units
|
||||
|
||||
|
||||
def backfill_reserve_reached() -> bool:
|
||||
"""True when today's remaining budget is at or below the backfill reserve — the shared yield
|
||||
condition for every backfill/explore path (recent, deep, immediate full-history, explore, and
|
||||
rolling revalidation).
|
||||
|
||||
Reads in its OWN short-lived session (like record_usage / log_action), for two reasons: it
|
||||
replaces the remaining/reserve comparison that was open-coded at every call site, and — critically
|
||||
for the per-page guards in paging loops — a guard must not hold the caller's transaction open
|
||||
across the network fetch that follows it. Quota is committed independently of any caller, so a
|
||||
fresh session reads the same numbers."""
|
||||
with SessionLocal() as qs:
|
||||
return remaining_today(qs) <= sysconfig.get_int(qs, "backfill_quota_reserve")
|
||||
|
||||
|
||||
def log_action(user_id: int, action: str) -> None:
|
||||
"""Record a zero-cost action event so per-user daily caps still count it.
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ from app.models import (
|
||||
Invite,
|
||||
User,
|
||||
)
|
||||
from app.routes.auth import revoke_preset_password_if_unverified
|
||||
from app.utils import iso, valid_email
|
||||
|
||||
router = APIRouter(prefix="/api/admin", tags=["admin"])
|
||||
@@ -62,6 +63,10 @@ def _activate_user_by_email(db: Session, email: str) -> None:
|
||||
it can sign in. No-op if no such user exists yet (a Google user is created at first login)."""
|
||||
user = db.execute(select(User).where(User.email == email.lower())).scalar_one_or_none()
|
||||
if user is not None and not user.is_active:
|
||||
# SEC-10: same guard as the Google-adoption path — approving an account whose email was never
|
||||
# verified must not hand a working login to a password set on it by whoever registered the
|
||||
# address (a pre-emption). Shared single-source helper so the two paths can't drift.
|
||||
revoke_preset_password_if_unverified(user)
|
||||
user.is_active = True
|
||||
db.commit()
|
||||
|
||||
|
||||
@@ -9,9 +9,9 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import AuditLog, User
|
||||
from app.routes.admin import admin_user
|
||||
from app.utils import iso
|
||||
|
||||
router = APIRouter(prefix="/api/admin/audit", tags=["admin"])
|
||||
|
||||
@@ -286,6 +286,9 @@ async def callback(
|
||||
return RedirectResponse(url="/?access=denied")
|
||||
user = existing
|
||||
user.google_sub = sub
|
||||
# SEC-10: drop any password pre-set on this account while its email was still unverified,
|
||||
# now that Google attests ownership (bumps the epoch to revoke any session it minted).
|
||||
revoke_preset_password_if_unverified(user)
|
||||
# Google verified the email and is_allowed granted access, so finish activating a
|
||||
# previously pending password registration (otherwise current_user would bounce it).
|
||||
user.email_verified = True
|
||||
@@ -392,8 +395,12 @@ def _complete_link(
|
||||
Google identity owned by another account, or to silently swap a different linked identity.
|
||||
`explicit` (the popped oauth_link_explicit marker) distinguishes a "Connect Google" click from
|
||||
a YouTube upgrade — the caller pops it up front so an aborted flow leaves nothing behind."""
|
||||
# The marker must match the live session user (guards a stale/forged marker).
|
||||
if link_uid != request.session.get("user_id"):
|
||||
# SEC-11: the marker must name an account signed into THIS browser — the session default OR any
|
||||
# account in the wallet — not only the default. A tab acting as a secondary account links Google to
|
||||
# ITSELF (the marker was set from current_user, which resolves the ?account= it navigated with), so
|
||||
# accepting wallet membership stops the identity/token landing on the default account A instead of B.
|
||||
accounts = request.session.get("account_ids") or []
|
||||
if link_uid != request.session.get("user_id") and link_uid not in accounts:
|
||||
return RedirectResponse(url="/?link=error")
|
||||
user = db.get(User, link_uid)
|
||||
if user is None or user.is_demo:
|
||||
@@ -805,6 +812,18 @@ def bump_session_epoch(user: User) -> None:
|
||||
user.session_epoch = (user.session_epoch or 0) + 1
|
||||
|
||||
|
||||
def revoke_preset_password_if_unverified(user: User) -> None:
|
||||
"""SEC-10 (account pre-emption): a password set on an account whose email was NEVER verified is
|
||||
untrusted — whoever registered the address (to pre-empt the real owner) could have chosen it. When
|
||||
ownership is finally attested (Google adoption, or admin approval), drop that password and bump the
|
||||
epoch to revoke any session it minted; the owner re-sets one via password-reset. No-op for an
|
||||
already-verified account, so a legit no-SMTP registration is unaffected. The single source for this
|
||||
rule — both the Google-callback and admin-activation paths call it so they can't drift."""
|
||||
if not user.email_verified:
|
||||
user.password_hash = None
|
||||
bump_session_epoch(user)
|
||||
|
||||
|
||||
@router.post("/logout-others")
|
||||
def logout_others(
|
||||
request: Request, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
|
||||
@@ -4,13 +4,13 @@ hidden, tags) and per-channel sync state. Read + local-write only — YouTube-si
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import quota, sysconfig
|
||||
from app.auth import current_user, require_human, require_write_scope
|
||||
from app import quota
|
||||
from app.auth import admin_user, current_user, require_human, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
@@ -26,9 +26,8 @@ from app.models import (
|
||||
Video,
|
||||
)
|
||||
from app.routes._common import owned_or_404
|
||||
from app.routes.admin import admin_user
|
||||
from app.sync.explore import explore_ingest_page
|
||||
from app.sync.runner import run_recent_backfill
|
||||
from app.sync.runner import backfill_channel_now, run_recent_backfill
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
@@ -50,20 +49,19 @@ def list_channels(
|
||||
).all()
|
||||
|
||||
# Per-channel aggregates over the shared catalog in ONE grouped pass: stored count, last
|
||||
# upload, total duration, and a Normal/Shorts/Live breakdown. Cheap — a single grouped scan
|
||||
# (~35 ms over the full 233k-row catalog on the dev DB), so computed on read rather than
|
||||
# upload, total duration. Cheap — a single grouped scan, so computed on read rather than
|
||||
# denormalised (measure-first: reads aren't frequent/hot enough to warrant cached columns).
|
||||
# (The Normal/Shorts/Live breakdown moved to the channel-detail page in R14, so it is no longer
|
||||
# computed here — the manager table dropped its N/S/L column.)
|
||||
channel_ids = [c.id for _, c in rows]
|
||||
agg: dict[str, dict] = {}
|
||||
if channel_ids:
|
||||
for cid, n, last_up, total_dur, shorts, live in db.execute(
|
||||
for cid, n, last_up, total_dur in db.execute(
|
||||
select(
|
||||
Video.channel_id,
|
||||
func.count(Video.id),
|
||||
func.max(Video.published_at),
|
||||
func.coalesce(func.sum(Video.duration_seconds), 0),
|
||||
func.count(Video.id).filter(Video.is_short.is_(True)),
|
||||
func.count(Video.id).filter(Video.live_status.in_(LIVE_OR_UPCOMING)),
|
||||
)
|
||||
.where(Video.channel_id.in_(channel_ids))
|
||||
.group_by(Video.channel_id)
|
||||
@@ -72,9 +70,6 @@ def list_channels(
|
||||
"stored": n,
|
||||
"last_video_at": iso(last_up),
|
||||
"total_duration_seconds": int(total_dur or 0),
|
||||
"count_short": int(shorts),
|
||||
"count_live": int(live),
|
||||
"count_normal": max(0, int(n) - int(shorts) - int(live)),
|
||||
}
|
||||
|
||||
# Channels already in the *global* deep-backfill queue — i.e. at least one user (any
|
||||
@@ -108,9 +103,6 @@ def list_channels(
|
||||
"stored_videos": (agg.get(ch.id) or {}).get("stored", 0),
|
||||
"last_video_at": (agg.get(ch.id) or {}).get("last_video_at"),
|
||||
"total_duration_seconds": (agg.get(ch.id) or {}).get("total_duration_seconds", 0),
|
||||
"count_normal": (agg.get(ch.id) or {}).get("count_normal", 0),
|
||||
"count_short": (agg.get(ch.id) or {}).get("count_short", 0),
|
||||
"count_live": (agg.get(ch.id) or {}).get("count_live", 0),
|
||||
"priority": sub.priority,
|
||||
"hidden": sub.hidden,
|
||||
"deep_requested": sub.deep_requested,
|
||||
@@ -229,6 +221,9 @@ def _channel_detail_dict(
|
||||
explored: bool,
|
||||
blocked: bool,
|
||||
stored: int,
|
||||
count_normal: int,
|
||||
count_short: int,
|
||||
count_live: int,
|
||||
deep_requested: bool,
|
||||
priority: int,
|
||||
tag_ids: list[int],
|
||||
@@ -253,6 +248,11 @@ def _channel_detail_dict(
|
||||
"explored": explored,
|
||||
"blocked": blocked,
|
||||
"stored_videos": stored,
|
||||
# Per-type breakdown (Normal / Shorts / Live) — R14 moved it off the manager table onto the
|
||||
# channel page. Same aggregation the manager list uses (count_normal = stored − shorts − live).
|
||||
"count_normal": count_normal,
|
||||
"count_short": count_short,
|
||||
"count_live": count_live,
|
||||
# Deep-backfill state, so the channel page can offer "get full history" when the catalog
|
||||
# holds only the recent-backfill/RSS subset (video_count is YouTube's raw total).
|
||||
"deep_requested": deep_requested,
|
||||
@@ -304,7 +304,19 @@ def channel_detail(
|
||||
)
|
||||
).first() is not None
|
||||
blocked = _is_blocked(db, user, channel_id)
|
||||
stored = db.scalar(select(func.count(Video.id)).where(Video.channel_id == channel_id)) or 0
|
||||
# Stored total + the Normal/Shorts/Live split in one grouped pass (same filters as the manager
|
||||
# list), so the channel page can show the per-type breakdown R14 moved off the table.
|
||||
n_all, n_short, n_live = db.execute(
|
||||
select(
|
||||
func.count(Video.id),
|
||||
func.count(Video.id).filter(Video.is_short.is_(True)),
|
||||
func.count(Video.id).filter(Video.live_status.in_(LIVE_OR_UPCOMING)),
|
||||
).where(Video.channel_id == channel_id)
|
||||
).one()
|
||||
stored = int(n_all or 0)
|
||||
count_short = int(n_short or 0)
|
||||
count_live = int(n_live or 0)
|
||||
count_normal = max(0, stored - count_short - count_live)
|
||||
tag_ids = list(
|
||||
db.scalars(
|
||||
select(ChannelTag.tag_id).where(
|
||||
@@ -318,6 +330,9 @@ def channel_detail(
|
||||
explored=explored,
|
||||
blocked=blocked,
|
||||
stored=int(stored),
|
||||
count_normal=count_normal,
|
||||
count_short=count_short,
|
||||
count_live=count_live,
|
||||
deep_requested=bool(sub.deep_requested) if sub else False,
|
||||
priority=int(sub.priority) if sub else 0,
|
||||
tag_ids=tag_ids,
|
||||
@@ -342,7 +357,7 @@ def explore_channel(
|
||||
raise HTTPException(status_code=404, detail="Unknown channel")
|
||||
if _is_blocked(db, user, channel_id):
|
||||
raise HTTPException(status_code=403, detail="You've blocked this channel.")
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
if quota.backfill_reserve_reached():
|
||||
raise HTTPException(
|
||||
status_code=429,
|
||||
detail="The shared daily quota reserve is reached — try exploring later.",
|
||||
@@ -426,6 +441,7 @@ class ChannelUpdateIn(BaseModel):
|
||||
def update_channel(
|
||||
channel_id: str,
|
||||
body: ChannelUpdateIn,
|
||||
background: BackgroundTasks,
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
@@ -450,12 +466,18 @@ def update_channel(
|
||||
|
||||
# Opting a channel into full history should give an immediate feed: if we haven't even
|
||||
# fetched its recent uploads yet, do that one channel now (cheap) instead of waiting for
|
||||
# the scheduler. Deep paging still follows on the scheduler's next run (recent-then-deep).
|
||||
# (deep_turned_on is only ever True for a non-demo user — see the guard above.)
|
||||
# the scheduler. (deep_turned_on is only ever True for a non-demo user — see the guard above.)
|
||||
if deep_turned_on and channel is not None and channel.recent_synced_at is None:
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_BACKFILL_RECENT):
|
||||
run_recent_backfill(db, [channel], max_channels=1)
|
||||
|
||||
# …and kick the FULL deep backfill for this one channel immediately in the background, so the
|
||||
# whole catalogue lands in seconds instead of over the scheduler's paced 10-minute ticks. The
|
||||
# request returns right away; the channel page's live poll shows the count climb. The scheduler
|
||||
# still finishes a very large channel's tail on its normal runs (deep_requested stays set).
|
||||
if deep_turned_on:
|
||||
background.add_task(backfill_channel_now, channel_id, user.id)
|
||||
|
||||
return {
|
||||
"id": channel_id,
|
||||
"priority": sub.priority,
|
||||
|
||||
@@ -11,9 +11,9 @@ from app import audit
|
||||
from app import email as email_mod
|
||||
from app import sysconfig
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import User
|
||||
from app.routes.admin import admin_user
|
||||
|
||||
router = APIRouter(prefix="/api/admin/config", tags=["admin"])
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ from app.db import get_db
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import links as linksmod
|
||||
from app.downloads import formats, quota, service, storage
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import (
|
||||
DownloadJob,
|
||||
DownloadLink,
|
||||
@@ -40,6 +41,11 @@ admin_router = APIRouter(prefix="/api/admin/downloads", tags=["admin-downloads"]
|
||||
|
||||
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
||||
|
||||
# Cap the per-user job listings so a heavy library can't return an unbounded result set. The
|
||||
# Download Center renders newest-first; 500 is far beyond any real manual-download history (and
|
||||
# matches the admin listing's own cap).
|
||||
_MAX_LIST = 500
|
||||
|
||||
|
||||
# --- source + serialization ----------------------------------------------------------------
|
||||
|
||||
@@ -334,6 +340,7 @@ def list_downloads(
|
||||
select(DownloadJob)
|
||||
.where(DownloadJob.user_id == user.id)
|
||||
.order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(_MAX_LIST)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -379,7 +386,8 @@ def shared_with_me(
|
||||
select(DownloadJob)
|
||||
.join(DownloadShare, DownloadShare.job_id == DownloadJob.id)
|
||||
.where(DownloadShare.to_user_id == user.id)
|
||||
.order_by(DownloadShare.created_at.desc())
|
||||
.order_by(DownloadShare.created_at.desc(), DownloadJob.id.desc())
|
||||
.limit(_MAX_LIST)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
@@ -537,9 +545,9 @@ def cancel_download(
|
||||
job_id: int, user: User = Depends(require_human), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
job = _own_job(db, user, job_id)
|
||||
if job.status not in ("done", "canceled"):
|
||||
if job.status not in (JobStatus.DONE, JobStatus.CANCELED):
|
||||
service.release_asset(db, job) # release while the job still counts as holding
|
||||
job.status = "canceled"
|
||||
job.status = JobStatus.CANCELED
|
||||
db.commit()
|
||||
return _serialize_job(db, job)
|
||||
|
||||
@@ -881,7 +889,7 @@ def admin_list_downloads(
|
||||
) -> list[dict]:
|
||||
jobs = (
|
||||
db.execute(
|
||||
select(DownloadJob).order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()).limit(500)
|
||||
select(DownloadJob).order_by(DownloadJob.created_at.desc(), DownloadJob.id.desc()).limit(_MAX_LIST)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
|
||||
+28
-426
@@ -1,29 +1,26 @@
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import re
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import BigInteger, Select, and_, cast, false, func, or_, select
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import StaleDataError
|
||||
|
||||
from app import quota
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.feedquery import (
|
||||
decode_cursor,
|
||||
encode_cursor,
|
||||
filtered_query,
|
||||
keyset_predicate,
|
||||
order_by,
|
||||
serialize,
|
||||
sort_keys,
|
||||
)
|
||||
from app.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
ChannelTag,
|
||||
ExploredChannel,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SearchFind,
|
||||
Subscription,
|
||||
Tag,
|
||||
User,
|
||||
Video,
|
||||
@@ -41,318 +38,12 @@ VALID_STATES = {"new", "watched", "hidden"}
|
||||
# Resume-position thresholds (mirror the client): positions below this are "didn't
|
||||
# really start" and within this of the end are "basically finished" — neither is worth
|
||||
# storing, so we clear the position instead (the video shows no progress bar).
|
||||
# NB: the Plex player intentionally uses a LARGER finish margin (30s) because it marks
|
||||
# long-form items watched, not just clears a resume bar — see routes/plex/playback.py.
|
||||
PROGRESS_MIN_SECONDS = 5
|
||||
FINISH_MARGIN_SECONDS = 10
|
||||
|
||||
|
||||
def _channel_url(channel_id: str, handle: str | None) -> str:
|
||||
if handle and handle.startswith("@"):
|
||||
return f"https://www.youtube.com/{handle}"
|
||||
return f"https://www.youtube.com/channel/{channel_id}"
|
||||
|
||||
|
||||
def _saved_expr(user: User):
|
||||
"""A correlated EXISTS labelled `saved`: is this video in the user's Watch later list?
|
||||
(Watch later is the built-in playlist that replaced the old per-video 'saved' status.)"""
|
||||
return (
|
||||
select(1)
|
||||
.select_from(PlaylistItem)
|
||||
.join(Playlist, Playlist.id == PlaylistItem.playlist_id)
|
||||
.where(
|
||||
Playlist.user_id == user.id,
|
||||
Playlist.kind == "watch_later",
|
||||
PlaylistItem.video_id == Video.id,
|
||||
)
|
||||
.exists()
|
||||
.label("saved")
|
||||
)
|
||||
|
||||
|
||||
def feed_columns(user: User, state) -> list:
|
||||
"""The shared feed-row projection (joined Channel fields + per-user watch state +
|
||||
`saved`), consumed by both the feed query and the playlist-detail query so a new column
|
||||
is added in exactly one place. `state` is the caller's aliased VideoState (outer-joined)."""
|
||||
return [
|
||||
Video.id,
|
||||
Video.title,
|
||||
Video.channel_id,
|
||||
Channel.title.label("channel_title"),
|
||||
Channel.thumbnail_url.label("channel_thumbnail"),
|
||||
Channel.handle.label("channel_handle"),
|
||||
Video.published_at,
|
||||
Video.thumbnail_url,
|
||||
Video.duration_seconds,
|
||||
Video.view_count,
|
||||
Video.is_short,
|
||||
Video.live_status,
|
||||
func.coalesce(state.status, "new").label("status"),
|
||||
func.coalesce(state.position_seconds, 0).label("position_seconds"),
|
||||
_saved_expr(user),
|
||||
]
|
||||
|
||||
|
||||
def serialize(row) -> dict:
|
||||
return {
|
||||
"id": row.id,
|
||||
"title": row.title,
|
||||
"channel_id": row.channel_id,
|
||||
"channel_title": row.channel_title,
|
||||
"channel_thumbnail": row.channel_thumbnail,
|
||||
"channel_url": _channel_url(row.channel_id, row.channel_handle),
|
||||
"published_at": iso(row.published_at),
|
||||
"thumbnail_url": row.thumbnail_url,
|
||||
"duration_seconds": row.duration_seconds,
|
||||
"view_count": row.view_count,
|
||||
"is_short": row.is_short,
|
||||
"live_status": row.live_status,
|
||||
"status": row.status or "new",
|
||||
"position_seconds": row.position_seconds or 0,
|
||||
"saved": bool(getattr(row, "saved", False)),
|
||||
"watch_url": f"https://www.youtube.com/watch?v={row.id}",
|
||||
}
|
||||
|
||||
|
||||
def _filtered_query(
|
||||
db: Session,
|
||||
user: User,
|
||||
*,
|
||||
tags: list[int],
|
||||
tag_mode: str,
|
||||
channel_ids: list[str],
|
||||
q: str | None,
|
||||
min_duration: int | None,
|
||||
max_duration: int | None,
|
||||
max_age_days: int | None,
|
||||
published_after: date | None,
|
||||
published_before: date | None,
|
||||
show_normal: bool,
|
||||
include_shorts: bool,
|
||||
include_live: bool,
|
||||
show: str,
|
||||
scope: str = "my",
|
||||
library_source: str = "organic",
|
||||
exclude_tag_category: str | None = None,
|
||||
) -> tuple[Select, object]:
|
||||
"""Build the feed query (joins + all WHERE filters), shared by /feed and /feed/count.
|
||||
Returns the column-bearing select plus the priority/relevance rank expression for sorting.
|
||||
|
||||
`scope="my"` (default) restricts the feed to the user's own non-hidden subscriptions.
|
||||
`scope="all"` shows every video in the shared catalog (any user's ingested channels);
|
||||
the subscription is then only LEFT-joined so per-channel priority still resolves for
|
||||
channels the user happens to be subscribed to. Per-user watch state stays private in
|
||||
either mode via the VideoState outer join."""
|
||||
state = aliased(VideoState)
|
||||
status_expr = func.coalesce(state.status, "new")
|
||||
position_expr = func.coalesce(state.position_seconds, 0)
|
||||
|
||||
query = select(*feed_columns(user, state)).join(
|
||||
Channel, Channel.id == Video.channel_id
|
||||
)
|
||||
|
||||
if scope == "all":
|
||||
# Whole shared catalog; subscription is optional (only for priority sort).
|
||||
query = query.outerjoin(
|
||||
Subscription,
|
||||
and_(
|
||||
Subscription.channel_id == Video.channel_id,
|
||||
Subscription.user_id == user.id,
|
||||
),
|
||||
)
|
||||
else:
|
||||
# "Mine": the user's own videos = their non-hidden subscriptions OR videos they
|
||||
# surfaced via their own live YouTube search. Both are LEFT-joined so the Source
|
||||
# filter below can include either or both, and the subscription row stays available
|
||||
# for the priority sort.
|
||||
query = query.outerjoin(
|
||||
Subscription,
|
||||
and_(
|
||||
Subscription.channel_id == Video.channel_id,
|
||||
Subscription.user_id == user.id,
|
||||
Subscription.hidden.is_(False),
|
||||
),
|
||||
).outerjoin(
|
||||
SearchFind,
|
||||
and_(SearchFind.video_id == Video.id, SearchFind.user_id == user.id),
|
||||
)
|
||||
|
||||
query = query.outerjoin(
|
||||
state, and_(state.video_id == Video.id, state.user_id == user.id)
|
||||
)
|
||||
|
||||
# Hide videos the maintenance job has flagged as currently unplayable (deleted/private/
|
||||
# abandoned). They're either deleted after a grace period or unhidden if they recover;
|
||||
# either way they shouldn't show in any feed view meanwhile.
|
||||
query = query.where(Video.unavailable_since.is_(None))
|
||||
|
||||
# Explored videos are PER-EXPLORER private: a via_explore video (ingested when someone
|
||||
# opened an un-subscribed channel's page) is visible only to users who explored or
|
||||
# subscribe to its channel, so one user's curiosity never leaks into everyone else's
|
||||
# catalog. Applied in BOTH scopes; the channel page reaches its ephemeral videos by
|
||||
# asking for scope=all + channel_ids (the channel is in this accessible set).
|
||||
accessible_explore = (
|
||||
select(Subscription.channel_id).where(Subscription.user_id == user.id)
|
||||
).union(
|
||||
select(ExploredChannel.channel_id).where(ExploredChannel.user_id == user.id)
|
||||
)
|
||||
query = query.where(
|
||||
or_(
|
||||
Video.via_explore.is_(False),
|
||||
Video.channel_id.in_(accessible_explore),
|
||||
)
|
||||
)
|
||||
|
||||
# Per-user channel blocklist: never show this user videos from a channel they blocked
|
||||
# (applies to feed, Library, explore, and the channel page alike).
|
||||
query = query.where(
|
||||
Video.channel_id.notin_(
|
||||
select(BlockedChannel.channel_id).where(BlockedChannel.user_id == user.id)
|
||||
)
|
||||
)
|
||||
|
||||
# Source filter. In the shared Library ("all" scope) it uses the GLOBAL via_search flag
|
||||
# (search-discovered by anyone). In "my" scope it's the user's OWN provenance:
|
||||
# "organic" = your subscriptions, "search" = videos you found via your own search,
|
||||
# "all" = both. Default "organic" keeps the Mine feed = subscriptions only.
|
||||
if scope == "all":
|
||||
if library_source == "organic":
|
||||
query = query.where(Video.via_search.is_(False))
|
||||
elif library_source == "search":
|
||||
query = query.where(Video.via_search.is_(True))
|
||||
# "all" → both organic and search-discovered videos
|
||||
else:
|
||||
subscribed = Subscription.channel_id.isnot(None)
|
||||
searched = SearchFind.video_id.isnot(None)
|
||||
if library_source == "search":
|
||||
query = query.where(searched)
|
||||
elif library_source == "all":
|
||||
query = query.where(or_(subscribed, searched))
|
||||
else: # "organic" (default): subscriptions only
|
||||
query = query.where(subscribed)
|
||||
|
||||
if channel_ids:
|
||||
# OR across the picked channels; every other filter still applies on top.
|
||||
query = query.where(Video.channel_id.in_(channel_ids))
|
||||
if min_duration is not None:
|
||||
query = query.where(Video.duration_seconds >= min_duration)
|
||||
if max_duration is not None:
|
||||
query = query.where(Video.duration_seconds <= max_duration)
|
||||
if max_age_days is not None:
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400
|
||||
query = query.where(
|
||||
Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)
|
||||
)
|
||||
if published_after is not None:
|
||||
start = datetime.combine(
|
||||
published_after, datetime.min.time(), tzinfo=timezone.utc
|
||||
)
|
||||
query = query.where(Video.published_at >= start)
|
||||
if published_before is not None:
|
||||
end = datetime.combine(
|
||||
published_before, datetime.min.time(), tzinfo=timezone.utc
|
||||
) + timedelta(days=1)
|
||||
query = query.where(Video.published_at < end)
|
||||
# Full-text relevance search over the weighted search document (title > creator keywords +
|
||||
# the queries that surfaced the video > description; see Video.search_vector). YouTube-like
|
||||
# "fuzzy" matching: word-order-independent, multi-word AND, prefix on the word being typed,
|
||||
# accent-insensitive (unaccent_simple). The channel name still matches as a substring so
|
||||
# channel searches work. `rank_expr` (weight-aware ts_rank) drives the "relevance" sort.
|
||||
rank_expr = None
|
||||
if q:
|
||||
ts_str = to_tsquery_str(q)
|
||||
if ts_str:
|
||||
tsq = func.to_tsquery("public.unaccent_simple", ts_str)
|
||||
query = query.where(
|
||||
or_(
|
||||
Video.search_vector.op("@@")(tsq),
|
||||
func.unaccent(Channel.title).ilike(func.unaccent(f"%{q}%")),
|
||||
)
|
||||
)
|
||||
# Scale ts_rank (a float4) to an integer score so the keyset cursor compares it
|
||||
# EXACTLY — a raw float4 vs the float8 cursor value mismatches on round-trip and
|
||||
# breaks paging (the same page repeats). 1e6 granularity is ample; ties break on
|
||||
# published_at then id.
|
||||
rank_expr = cast(func.ts_rank(Video.search_vector, tsq) * 1000000, BigInteger)
|
||||
else:
|
||||
# No usable tokens (e.g. only punctuation): fall back to a plain substring match.
|
||||
like = func.unaccent(f"%{q}%")
|
||||
query = query.where(
|
||||
or_(
|
||||
func.unaccent(Video.title).ilike(like),
|
||||
func.unaccent(Channel.title).ilike(like),
|
||||
)
|
||||
)
|
||||
|
||||
if tags:
|
||||
# AND across tag categories (e.g. language AND topic narrows), OR within a
|
||||
# category; the any/all toggle controls multiple topic tags.
|
||||
cat_rows = db.execute(select(Tag.id, Tag.category).where(Tag.id.in_(tags))).all()
|
||||
by_category: dict[str, list[int]] = {}
|
||||
for tag_id, category in cat_rows:
|
||||
by_category.setdefault(category, []).append(tag_id)
|
||||
# Facet counting drops the category being counted so its own chips don't zero each
|
||||
# other out (standard drill-down faceting: a category's count ignores its own
|
||||
# selections but still honours the other categories' filters).
|
||||
if exclude_tag_category:
|
||||
by_category.pop(exclude_tag_category, None)
|
||||
visible = or_(ChannelTag.user_id.is_(None), ChannelTag.user_id == user.id)
|
||||
for category, ids in by_category.items():
|
||||
if category == "topic" and tag_mode == "and" and len(ids) > 1:
|
||||
sub = (
|
||||
select(ChannelTag.channel_id)
|
||||
.where(ChannelTag.tag_id.in_(ids), visible)
|
||||
.group_by(ChannelTag.channel_id)
|
||||
.having(func.count(func.distinct(ChannelTag.tag_id)) == len(set(ids)))
|
||||
)
|
||||
else:
|
||||
sub = select(ChannelTag.channel_id).where(
|
||||
ChannelTag.tag_id.in_(ids), visible
|
||||
)
|
||||
query = query.where(Video.channel_id.in_(sub))
|
||||
|
||||
# Content type: Normal / Shorts / Live·Upcoming as a union of enabled types.
|
||||
# Explicit Watched/Saved/Hidden views show every type so nothing goes missing.
|
||||
explicit_view = show in ("watched", "hidden")
|
||||
if not explicit_view:
|
||||
type_clauses = []
|
||||
if show_normal:
|
||||
type_clauses.append(
|
||||
and_(Video.is_short.is_(False), Video.live_status.notin_(LIVE_OR_UPCOMING))
|
||||
)
|
||||
if include_shorts:
|
||||
type_clauses.append(Video.is_short.is_(True))
|
||||
if include_live:
|
||||
type_clauses.append(Video.live_status.in_(LIVE_OR_UPCOMING))
|
||||
query = query.where(or_(*type_clauses) if type_clauses else false())
|
||||
|
||||
if show == "unwatched":
|
||||
query = query.where(status_expr.notin_(("watched", "hidden")))
|
||||
elif show == "in_progress":
|
||||
# Started but not finished: a stored resume position, not yet watched/hidden.
|
||||
query = query.where(
|
||||
and_(position_expr > 0, status_expr.notin_(("watched", "hidden")))
|
||||
)
|
||||
elif show == "watched":
|
||||
query = query.where(status_expr == "watched")
|
||||
elif show == "hidden":
|
||||
query = query.where(status_expr == "hidden")
|
||||
else: # all
|
||||
query = query.where(status_expr != "hidden")
|
||||
|
||||
return query, rank_expr
|
||||
|
||||
|
||||
def to_tsquery_str(q: str) -> str | None:
|
||||
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
|
||||
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*".
|
||||
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
|
||||
(the string is built from sanitised tokens, never the raw query)."""
|
||||
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
|
||||
if not tokens:
|
||||
return None
|
||||
tokens[-1] = tokens[-1] + ":*"
|
||||
return " & ".join(tokens)
|
||||
|
||||
|
||||
# Shared query parameters for /feed and /feed/count.
|
||||
def _feed_params(
|
||||
tags: list[int] = Query(default=[]),
|
||||
@@ -390,101 +81,6 @@ def _feed_params(
|
||||
}
|
||||
|
||||
|
||||
# --- Keyset (cursor) pagination ---------------------------------------------
|
||||
#
|
||||
# Each sort is described as an ordered list of key columns (expression + direction
|
||||
# + whether it can be NULL + a JSON round-trip kind), always with Video.id as the
|
||||
# final, unique tiebreaker. This drives BOTH the ORDER BY and the keyset WHERE so
|
||||
# they can never drift apart — the prerequisite for correct cursor paging. Keyset
|
||||
# replaces OFFSET/LIMIT: deep scrolling no longer scans-and-discards skipped rows,
|
||||
# and the page boundary is stable even as the scheduler ingests new videos mid-scroll
|
||||
# (OFFSET would shift and duplicate/skip rows).
|
||||
def _sort_keys(sort: str, seed: int, rank_expr=None) -> list[dict]:
|
||||
pub = {"expr": Video.published_at, "asc": False, "nullable": True, "kind": "dt"}
|
||||
table: dict[str, list[dict]] = {
|
||||
"newest": [pub],
|
||||
"oldest": [{**pub, "asc": True}],
|
||||
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
|
||||
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
|
||||
"duration_desc": [{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}],
|
||||
"duration_asc": [{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}],
|
||||
"title": [{"expr": func.lower(Video.title), "asc": True, "nullable": True, "kind": "str"}],
|
||||
"title_desc": [{"expr": func.lower(Video.title), "asc": False, "nullable": True, "kind": "str"}],
|
||||
"subscribers": [{"expr": Channel.subscriber_count, "asc": False, "nullable": True, "kind": "num"}],
|
||||
"subscribers_asc": [{"expr": Channel.subscriber_count, "asc": True, "nullable": True, "kind": "num"}],
|
||||
# Your per-channel priority (set in the channel manager), newest first within a tier.
|
||||
# coalesce keeps it null-safe in "all" scope where unsubscribed channels have no row.
|
||||
"priority": [{"expr": func.coalesce(Subscription.priority, 0), "asc": False, "nullable": False, "kind": "num"}, pub],
|
||||
"priority_asc": [{"expr": func.coalesce(Subscription.priority, 0), "asc": True, "nullable": False, "kind": "num"}, pub],
|
||||
# Deterministic per-seed shuffle: the same seed yields the same total order, so
|
||||
# the cursor stays valid across pages of one shuffled run.
|
||||
"shuffle": [{"expr": func.md5(func.concat(Video.id, str(seed))), "asc": True, "nullable": False, "kind": "str"}],
|
||||
}
|
||||
# Relevance (search-result ranking) only exists when there's a query to rank against;
|
||||
# without `rank_expr` it falls through to newest. Ties break by newest, then id.
|
||||
if rank_expr is not None:
|
||||
table["relevance"] = [
|
||||
{"expr": rank_expr, "asc": False, "nullable": False, "kind": "num"},
|
||||
pub,
|
||||
]
|
||||
return table.get(sort) or table["newest"]
|
||||
|
||||
|
||||
def _order_by(keys: list[dict]):
|
||||
cols = []
|
||||
for k in keys:
|
||||
c = k["expr"].asc() if k["asc"] else k["expr"].desc()
|
||||
cols.append(c.nulls_last() if k["nullable"] else c)
|
||||
cols.append(Video.id.asc()) # unique final tiebreaker
|
||||
return cols
|
||||
|
||||
|
||||
def _keyset_predicate(keys: list[dict], cursor_id: str):
|
||||
"""WHERE clause selecting rows strictly AFTER the cursor in the sort order.
|
||||
|
||||
Lexicographic over the key columns then id, NULL-aware for `nulls_last`: a NULL
|
||||
sorts after every non-NULL value, so a cursor sitting on a non-NULL value must
|
||||
also pull in the NULL group, and equality treats NULL == NULL."""
|
||||
def eq(k):
|
||||
return k["expr"].is_(None) if k["value"] is None else k["expr"] == k["value"]
|
||||
|
||||
def after(k):
|
||||
if k["value"] is None:
|
||||
return false() # nulls are last; nothing sorts strictly after them
|
||||
cmp = k["expr"] > k["value"] if k["asc"] else k["expr"] < k["value"]
|
||||
return or_(cmp, k["expr"].is_(None)) if k["nullable"] else cmp
|
||||
|
||||
branches = []
|
||||
for j in range(len(keys)):
|
||||
branches.append(and_(*[eq(keys[i]) for i in range(j)], after(keys[j])))
|
||||
branches.append(and_(*[eq(k) for k in keys], Video.id > cursor_id))
|
||||
return or_(*branches)
|
||||
|
||||
|
||||
def _encode_cursor(keys: list[dict], row) -> str:
|
||||
vals = []
|
||||
for i, k in enumerate(keys):
|
||||
v = getattr(row, f"_k{i}")
|
||||
vals.append(v.isoformat() if (v is not None and k["kind"] == "dt") else v)
|
||||
payload = json.dumps({"k": vals, "id": row.id})
|
||||
return base64.urlsafe_b64encode(payload.encode()).decode()
|
||||
|
||||
|
||||
def _decode_cursor(keys: list[dict], cursor: str) -> tuple[list, str]:
|
||||
try:
|
||||
payload = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
raw = payload["k"]
|
||||
if len(raw) != len(keys):
|
||||
raise ValueError("key arity mismatch")
|
||||
vals = [
|
||||
datetime.fromisoformat(v) if (v is not None and k["kind"] == "dt") else v
|
||||
for k, v in zip(keys, raw)
|
||||
]
|
||||
return vals, payload["id"]
|
||||
except (binascii.Error, json.JSONDecodeError, KeyError, ValueError, TypeError):
|
||||
raise HTTPException(status_code=400, detail="Invalid feed cursor")
|
||||
|
||||
|
||||
@router.get("/feed")
|
||||
def get_feed(
|
||||
params: dict = Depends(_feed_params),
|
||||
@@ -495,25 +91,31 @@ def get_feed(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
query, rank_expr = _filtered_query(db, user, **params)
|
||||
keys = _sort_keys(sort, seed, rank_expr)
|
||||
query, rank_expr = filtered_query(db, user, **params)
|
||||
keys = sort_keys(sort, seed, rank_expr)
|
||||
|
||||
# Expose each key column on the row so we can build the next cursor from the last item.
|
||||
query = query.add_columns(*[k["expr"].label(f"_k{i}") for i, k in enumerate(keys)])
|
||||
query = query.order_by(*_order_by(keys))
|
||||
query = query.order_by(*order_by(keys))
|
||||
|
||||
if cursor:
|
||||
vals, cursor_id = _decode_cursor(keys, cursor)
|
||||
# The read-model raises a plain ValueError on a malformed cursor; translate it to the
|
||||
# HTTP 400 here so feedquery stays web-framework-free.
|
||||
try:
|
||||
vals, cursor_id = decode_cursor(keys, cursor)
|
||||
except ValueError:
|
||||
# `from None`: the ValueError is an expected bad-input signal, not an error to chain.
|
||||
raise HTTPException(status_code=400, detail="Invalid feed cursor") from None
|
||||
for k, v in zip(keys, vals):
|
||||
k["value"] = v
|
||||
query = query.where(_keyset_predicate(keys, cursor_id))
|
||||
query = query.where(keyset_predicate(keys, cursor_id))
|
||||
|
||||
rows = db.execute(query.limit(limit + 1)).all()
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
return {
|
||||
"items": [serialize(r) for r in page],
|
||||
"next_cursor": _encode_cursor(keys, page[-1]) if (has_more and page) else None,
|
||||
"next_cursor": encode_cursor(keys, page[-1]) if (has_more and page) else None,
|
||||
"has_more": has_more,
|
||||
"limit": limit,
|
||||
}
|
||||
@@ -525,7 +127,7 @@ def get_feed_count(
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
query, _rank = _filtered_query(db, user, **params)
|
||||
query, _rank = filtered_query(db, user, **params)
|
||||
total = db.scalar(select(func.count()).select_from(query.subquery()))
|
||||
return {"count": total or 0}
|
||||
|
||||
@@ -553,7 +155,7 @@ def get_facets(
|
||||
# keeps them applied, so each remaining chip narrows to channels that ALSO have all
|
||||
# already-selected topics — and tags that can't co-occur drop to zero (hidden).
|
||||
conjunctive = category == "topic" and params.get("tag_mode") == "and"
|
||||
base, _rank = _filtered_query(
|
||||
base, _rank = filtered_query(
|
||||
db,
|
||||
user,
|
||||
**{**params, "exclude_tag_category": None if conjunctive else category},
|
||||
|
||||
@@ -72,6 +72,18 @@ def switch_account(
|
||||
# Don't make a suspended account the active one — the next request's current_user would
|
||||
# see the suspension and clear the WHOLE wallet session, logging out every account here.
|
||||
raise HTTPException(status_code=403, detail="That account is suspended.")
|
||||
# Same reasoning for a REVOKED session (a password reset/change or "log out everywhere" bumped the
|
||||
# account's epoch): making it the session default would trip current_user's epoch check on the next
|
||||
# request and clear the whole wallet. Mirror the suspend guard — drop the stale account from the
|
||||
# switch list and refuse, rather than logging everyone out. (B-core-9)
|
||||
epochs = request.session.get("epochs") or {}
|
||||
if int(epochs.get(str(target), 0)) != (u.session_epoch or 0):
|
||||
request.session["account_ids"] = [
|
||||
a for a in (request.session.get("account_ids") or []) if a != target
|
||||
]
|
||||
raise HTTPException(
|
||||
status_code=403, detail="That account was signed out elsewhere — sign in again."
|
||||
)
|
||||
request.session["user_id"] = target
|
||||
return {"ok": True, "user_id": target}
|
||||
|
||||
|
||||
@@ -9,8 +9,8 @@ from sqlalchemy.orm import Session, aliased
|
||||
from app import quota
|
||||
from app.auth import current_user, has_read_scope, has_write_scope, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import Channel, Playlist, PlaylistItem, User, Video, VideoState
|
||||
from app.routes.feed import serialize, feed_columns
|
||||
from app.sync.playlists import (
|
||||
plan_push,
|
||||
push_playlist,
|
||||
|
||||
@@ -8,12 +8,13 @@ from sqlalchemy.orm import Session
|
||||
from app.models import (
|
||||
PlexItem,
|
||||
)
|
||||
from app.plex.lookups import by_rating_key
|
||||
|
||||
log = logging.getLogger("siftlode.plex")
|
||||
|
||||
|
||||
def item_or_404(db: Session, rating_key: str) -> PlexItem:
|
||||
it = db.query(PlexItem).filter_by(rating_key=str(rating_key)).first()
|
||||
it = by_rating_key(db, PlexItem, rating_key)
|
||||
if it is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex item")
|
||||
return it
|
||||
|
||||
@@ -4,10 +4,11 @@ from datetime import datetime, timezone
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import sysconfig
|
||||
from app.auth import current_user
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
PlexItem,
|
||||
@@ -18,7 +19,6 @@ from app.plex import paths as plex_paths
|
||||
from app.plex import sync as plex_sync
|
||||
from app.plex import watch_sync as plex_watch
|
||||
from app.plex.client import PlexClient, PlexError, PlexNotConfigured
|
||||
from app.routes.admin import admin_user
|
||||
from app.utils import iso
|
||||
|
||||
router = APIRouter(prefix="/api/plex", tags=["plex"])
|
||||
@@ -65,9 +65,9 @@ def _sample_plex_path(
|
||||
"""A real Plex-side media file path to validate the mount + path-map against. Prefers the
|
||||
already-mirrored catalog (cheap); before the first sync, pulls one movie leaf from an enabled
|
||||
section straight from Plex."""
|
||||
row = db.query(PlexItem.file_path).filter(PlexItem.file_path.isnot(None)).first()
|
||||
if row and row[0]:
|
||||
return row[0]
|
||||
fp = db.scalar(select(PlexItem.file_path).where(PlexItem.file_path.isnot(None)))
|
||||
if fp:
|
||||
return fp
|
||||
wanted = plex_sync._enabled_section_keys(db)
|
||||
for s in sections:
|
||||
if s.get("type") != "movie":
|
||||
@@ -117,7 +117,9 @@ def watch_link_status(
|
||||
user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""This user's Plex watch-sync link status (drives the Settings toggle)."""
|
||||
link = db.query(PlexLink).filter_by(user_id=user.id).first()
|
||||
link = db.execute(
|
||||
select(PlexLink).where(PlexLink.user_id == user.id)
|
||||
).scalars().first()
|
||||
return _link_dict(link)
|
||||
|
||||
|
||||
@@ -137,7 +139,9 @@ def watch_link_set(
|
||||
if not sysconfig.get_bool(db, "plex_enabled"):
|
||||
raise HTTPException(status_code=409, detail="The Plex module is disabled.")
|
||||
enabled = body.enabled
|
||||
link = db.query(PlexLink).filter_by(user_id=user.id).first()
|
||||
link = db.execute(
|
||||
select(PlexLink).where(PlexLink.user_id == user.id)
|
||||
).scalars().first()
|
||||
if link is None:
|
||||
link = PlexLink(
|
||||
user_id=user.id, uses_admin=True, linked_at=datetime.now(timezone.utc)
|
||||
@@ -158,7 +162,9 @@ def watch_import_now(
|
||||
) -> dict:
|
||||
"""Admin: (re-)run the "Plex is master" import for this account — Plex re-wins on the
|
||||
intersection; Siftlode-only states are preserved. Idempotent."""
|
||||
link = db.query(PlexLink).filter_by(user_id=user.id).first()
|
||||
link = db.execute(
|
||||
select(PlexLink).where(PlexLink.user_id == user.id)
|
||||
).scalars().first()
|
||||
if link is None or not link.sync_enabled:
|
||||
raise HTTPException(status_code=409, detail="Enable Plex watch-sync first.")
|
||||
result = plex_watch.import_owner_watch_state(db, link)
|
||||
|
||||
+555
-523
File diff suppressed because it is too large
Load Diff
@@ -16,6 +16,7 @@ from fastapi import (
|
||||
)
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.auth import current_user, resolved_user_id
|
||||
@@ -29,6 +30,7 @@ from app.models import (
|
||||
)
|
||||
from app.plex import imgcache, subtitles
|
||||
from app.plex import paths as plex_paths
|
||||
from app.plex.lookups import by_rating_key
|
||||
from app.plex import people as plex_people
|
||||
from app.plex import stream as plex_stream
|
||||
from app.plex import watch_sync as plex_watch
|
||||
@@ -158,6 +160,11 @@ def subtitle(
|
||||
return Response(subtitles.shift_vtt(vtt, offset), media_type="text/vtt")
|
||||
|
||||
|
||||
# Resume/watched thresholds. _PROGRESS_MIN_S matches the feed's PROGRESS_MIN_SECONDS (5), but
|
||||
# _FINISH_MARGIN_S is DELIBERATELY larger than the feed's FINISH_MARGIN_SECONDS (30 vs 10): this is
|
||||
# long-form content (movies/episodes with credits) and crossing it MARKS THE ITEM WATCHED, whereas
|
||||
# the feed's margin only clears a short video's resume bar — different domains, intentional divergence,
|
||||
# not drift. Keep them separate on purpose (see routes/feed.py).
|
||||
_PROGRESS_MIN_S = 5
|
||||
|
||||
_FINISH_MARGIN_S = 30
|
||||
@@ -202,7 +209,11 @@ def _push_watch(
|
||||
return
|
||||
# Capture the row's freshest timestamp NOW so the background push can tell if a newer local edit
|
||||
# lands before it settles synced_to_plex (which would otherwise strand that newer value).
|
||||
st = db.query(PlexState).filter_by(user_id=user_id, item_id=item_id).first()
|
||||
st = db.execute(
|
||||
select(PlexState).where(
|
||||
PlexState.user_id == user_id, PlexState.item_id == item_id
|
||||
)
|
||||
).scalars().first()
|
||||
background.add_task(
|
||||
plex_watch.push_state_to_plex,
|
||||
user_id,
|
||||
@@ -241,7 +252,11 @@ def item_progress(
|
||||
duration = int(body.duration_seconds)
|
||||
final = body.final
|
||||
now = datetime.now(timezone.utc)
|
||||
st = db.query(PlexState).filter_by(user_id=user.id, item_id=it.id).first()
|
||||
st = db.execute(
|
||||
select(PlexState).where(
|
||||
PlexState.user_id == user.id, PlexState.item_id == it.id
|
||||
)
|
||||
).scalars().first()
|
||||
was_watched = st is not None and st.status == "watched"
|
||||
|
||||
near_end = duration > 0 and position > duration - _FINISH_MARGIN_S
|
||||
@@ -311,7 +326,11 @@ def item_state(
|
||||
status = body.status
|
||||
if status not in ("new", "watched", "hidden"):
|
||||
raise HTTPException(status_code=400, detail="Invalid status")
|
||||
st = db.query(PlexState).filter_by(user_id=user.id, item_id=it.id).first()
|
||||
st = db.execute(
|
||||
select(PlexState).where(
|
||||
PlexState.user_id == user.id, PlexState.item_id == it.id
|
||||
)
|
||||
).scalars().first()
|
||||
prev_status = st.status if st is not None else "new"
|
||||
if status == "new":
|
||||
if st is not None:
|
||||
@@ -346,10 +365,12 @@ def _bulk_state(
|
||||
now = datetime.now(timezone.utc)
|
||||
existing = {
|
||||
s.item_id: s
|
||||
for s in db.query(PlexState).filter(
|
||||
PlexState.user_id == user_id,
|
||||
PlexState.item_id.in_([e.id for e in eps] or [0]),
|
||||
)
|
||||
for s in db.execute(
|
||||
select(PlexState).where(
|
||||
PlexState.user_id == user_id,
|
||||
PlexState.item_id.in_([e.id for e in eps] or [0]),
|
||||
)
|
||||
).scalars()
|
||||
}
|
||||
pushable = plex_watch.link_for_push(db, user_id) is not None
|
||||
to_push: list[tuple[int, str, str]] = []
|
||||
@@ -392,11 +413,13 @@ def show_state(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Mark a whole show watched/unwatched (all its episodes) for the current user + push to Plex."""
|
||||
sh = db.query(PlexShow).filter_by(rating_key=str(rating_key)).first()
|
||||
sh = by_rating_key(db, PlexShow, rating_key)
|
||||
if sh is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex show")
|
||||
watched = body.watched
|
||||
eps = db.query(PlexItem).filter_by(show_id=sh.id, kind="episode").all()
|
||||
eps = db.execute(
|
||||
select(PlexItem).where(PlexItem.show_id == sh.id, PlexItem.kind == "episode")
|
||||
).scalars().all()
|
||||
changed = _bulk_state(background, db, user.id, eps, watched)
|
||||
return {"changed": changed, "watched": watched}
|
||||
|
||||
@@ -410,11 +433,13 @@ def season_state(
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Mark a whole season watched/unwatched (all its episodes) for the current user + push to Plex."""
|
||||
se = db.query(PlexSeason).filter_by(rating_key=str(rating_key)).first()
|
||||
se = by_rating_key(db, PlexSeason, rating_key)
|
||||
if se is None:
|
||||
raise HTTPException(status_code=404, detail="Unknown Plex season")
|
||||
watched = body.watched
|
||||
eps = db.query(PlexItem).filter_by(season_id=se.id, kind="episode").all()
|
||||
eps = db.execute(
|
||||
select(PlexItem).where(PlexItem.season_id == se.id, PlexItem.kind == "episode")
|
||||
).scalars().all()
|
||||
changed = _bulk_state(background, db, user.id, eps, watched)
|
||||
return {"changed": changed, "watched": watched}
|
||||
|
||||
|
||||
@@ -10,10 +10,9 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import quota
|
||||
from app.auth import current_user
|
||||
from app.auth import admin_user, current_user
|
||||
from app.db import get_db
|
||||
from app.models import QuotaEvent, User
|
||||
from app.routes.admin import admin_user
|
||||
|
||||
router = APIRouter(prefix="/api/quota", tags=["quota"])
|
||||
|
||||
|
||||
@@ -8,10 +8,10 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import audit, quota, state, sysconfig
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
from app.routes.admin import admin_user
|
||||
from app.scheduler import (
|
||||
JOB_INTERVALS,
|
||||
MAX_INTERVAL,
|
||||
|
||||
@@ -36,7 +36,7 @@ from app.models import (
|
||||
Video,
|
||||
VideoState,
|
||||
)
|
||||
from app.routes.feed import serialize, feed_columns
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.sync.subscriptions import apply_channel_details
|
||||
from app.sync.videos import insert_stubs, apply_video_details, parse_dt
|
||||
from app.youtube import search_scrape
|
||||
@@ -268,7 +268,11 @@ def search_youtube(
|
||||
except (YouTubeError, search_scrape.ScrapeError) as exc:
|
||||
if collected:
|
||||
break # keep what we already gathered
|
||||
raise HTTPException(status_code=502, detail=f"YouTube search failed: {exc}")
|
||||
# 422, not 502: this is a JSON endpoint, and the FE treats 502/503/504 as a gateway
|
||||
# dropout (auto-retry + "connection lost" banner — lib/api/core RETRIABLE_GATEWAY),
|
||||
# which hides the real message and burns a second quota-costed search. Upstream failure
|
||||
# on a JSON endpoint = 422 + message; reserve 5xx for the raw byte-proxies. (B-core-1)
|
||||
raise HTTPException(status_code=422, detail=f"YouTube search failed: {exc}") from exc
|
||||
|
||||
# Drop currently-live/upcoming, channel-less, and blocked-channel results.
|
||||
candidates = [
|
||||
|
||||
@@ -25,7 +25,7 @@ from datetime import timedelta
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, quota, sysconfig
|
||||
from app import progress, quota
|
||||
from app.config import settings
|
||||
from app.models import Playlist, PlaylistItem, Video
|
||||
from app.notifications import create_notification
|
||||
@@ -188,7 +188,7 @@ def _revalidate_rolling(db: Session, yt: YouTubeClient) -> dict:
|
||||
now = _now()
|
||||
# Process in videos.list-sized pages so we commit incrementally and stop on low quota.
|
||||
while checked < batch:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
if quota.backfill_reserve_reached():
|
||||
break
|
||||
page = (
|
||||
db.execute(
|
||||
@@ -228,7 +228,7 @@ def run_maintenance(db: Session) -> dict:
|
||||
phase1 = _recheck_flagged(db, yt)
|
||||
phase2 = (
|
||||
_revalidate_rolling(db, yt)
|
||||
if quota.remaining_today(db) > sysconfig.get_int(db, "backfill_quota_reserve")
|
||||
if not quota.backfill_reserve_reached()
|
||||
else {"checked": 0, "flagged": 0, "skipped": "low quota"}
|
||||
)
|
||||
return {
|
||||
|
||||
@@ -5,6 +5,7 @@ user" (any user with a stored refresh token) unless a YOUTUBE_API_KEY is configu
|
||||
public reads.
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
|
||||
from sqlalchemy import select
|
||||
@@ -13,6 +14,7 @@ from sqlalchemy.orm import Session
|
||||
from app import progress, quota, sysconfig
|
||||
|
||||
log = logging.getLogger("siftlode.sync")
|
||||
from app.db import SessionLocal
|
||||
from app.models import Channel, OAuthToken, Subscription, User
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
from app.sync.videos import (
|
||||
@@ -181,7 +183,7 @@ def run_recent_backfill(
|
||||
videos_added = 0
|
||||
with YouTubeClient(db, user) as yt:
|
||||
for channel in channels:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
if quota.backfill_reserve_reached():
|
||||
break
|
||||
try:
|
||||
videos_added += backfill_channel_recent(db, yt, channel)
|
||||
@@ -227,8 +229,14 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
videos_added = 0
|
||||
with YouTubeClient(db, user) as yt:
|
||||
for channel in channels:
|
||||
if quota.remaining_today(db) <= sysconfig.get_int(db, "backfill_quota_reserve"):
|
||||
if quota.backfill_reserve_reached():
|
||||
break
|
||||
# Claim the channel (or skip it if an interactive backfill already owns it) so the two
|
||||
# never advance the same cursor concurrently — symmetric with backfill_channel_now.
|
||||
with _backfill_lock:
|
||||
if channel.id in _backfill_in_progress:
|
||||
continue
|
||||
_backfill_in_progress.add(channel.id)
|
||||
try:
|
||||
videos_added += backfill_channel_deep(db, yt, channel, max_pages)
|
||||
processed += 1
|
||||
@@ -236,6 +244,9 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
except Exception:
|
||||
db.rollback()
|
||||
log.exception("Deep backfill failed for channel %s", channel.id)
|
||||
finally:
|
||||
with _backfill_lock:
|
||||
_backfill_in_progress.discard(channel.id)
|
||||
reconciled = reconcile_full_history(db)
|
||||
return {
|
||||
"channels_processed": processed,
|
||||
@@ -244,6 +255,63 @@ def run_deep_backfill(db: Session, max_channels: int = 10, max_pages: int = 5) -
|
||||
}
|
||||
|
||||
|
||||
# Channels being deep-backfilled RIGHT NOW by EITHER path — the interactive backfill_channel_now (a
|
||||
# FastAPI BackgroundTasks threadpool thread) OR the scheduler's run_deep_backfill (its own thread).
|
||||
# BOTH register the channel they're paging (under the lock) so the other skips it: no two runs page
|
||||
# the same channel's `backfill_cursor` concurrently, whichever started first. `_immediate_running`
|
||||
# counts only the interactive path, capped so one user's burst can't drain the shared budget (the
|
||||
# scheduler is already quota-paced). Single uvicorn process → one shared module state, lock suffices.
|
||||
_backfill_in_progress: set[str] = set()
|
||||
_immediate_running = 0
|
||||
_backfill_lock = threading.Lock()
|
||||
_MAX_CONCURRENT_IMMEDIATE = 2
|
||||
|
||||
|
||||
def backfill_channel_now(channel_id: str, user_id: int, max_pages: int = 400) -> None:
|
||||
"""User-initiated "get full history": deep-backfill ONE channel to completion right now, in a
|
||||
background task with its OWN session, so the full catalogue lands in seconds instead of over the
|
||||
paced scheduler's 10-minute ticks (max_pages=5/channel/tick). max_pages is high enough to finish
|
||||
any realistic channel in a single run (400 pages = 20 000 videos; e.g. BBC Earth's ~3400 = 69
|
||||
pages — the old 60-page cap stopped one page short of large channels, so their tail dribbled in
|
||||
over the scheduler's 10-minute ticks and looked stuck). backfill_channel_deep re-checks the shared
|
||||
budget before each page and yields at the reserve, so a high cap still can't drain the day's quota.
|
||||
Quota-guarded, capped to _MAX_CONCURRENT_IMMEDIATE interactive runs so one user can't drain the
|
||||
shared backfill budget by opting many channels in at once (the overflow, and any channel too large
|
||||
to finish under the reserve, is finished by the scheduler since deep_requested stays set), and
|
||||
mutually excluded per-channel with the scheduler via _backfill_in_progress. Best-effort — logs and
|
||||
swallows failures."""
|
||||
global _immediate_running
|
||||
with _backfill_lock:
|
||||
if channel_id in _backfill_in_progress or _immediate_running >= _MAX_CONCURRENT_IMMEDIATE:
|
||||
return # already backfilling this channel, or too many interactive runs — leave it to the scheduler
|
||||
_backfill_in_progress.add(channel_id)
|
||||
_immediate_running += 1
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
user = get_service_user(db)
|
||||
if user is None:
|
||||
return
|
||||
channel = db.get(Channel, channel_id)
|
||||
if channel is None or channel.backfill_done or not channel.uploads_playlist_id:
|
||||
return
|
||||
if quota.backfill_reserve_reached():
|
||||
return
|
||||
try:
|
||||
# Attribute the spend to the requesting user under the full-backfill action, so it
|
||||
# shows up correctly in the usage breakdown (not the default OTHER/system bucket).
|
||||
with quota.attribute(user_id, quota.QuotaAction.VIDEOS_BACKFILL_FULL):
|
||||
with YouTubeClient(db, user) as yt:
|
||||
backfill_channel_deep(db, yt, channel, max_pages)
|
||||
reconcile_full_history(db)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
log.exception("Immediate backfill failed for channel %s", channel_id)
|
||||
finally:
|
||||
with _backfill_lock:
|
||||
_backfill_in_progress.discard(channel_id)
|
||||
_immediate_running -= 1
|
||||
|
||||
|
||||
def estimate_deep_backfill(db: Session, channels: list[Channel]) -> dict:
|
||||
"""Rough ETA to finish full-history backfill of `channels`. Quota-bound: total
|
||||
estimated API units / the daily budget left for backfill. Deliberately approximate —
|
||||
|
||||
@@ -10,7 +10,7 @@ from sqlalchemy import and_, func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import progress, sysconfig
|
||||
from app import progress, quota, sysconfig
|
||||
from app.models import LIVE_OR_UPCOMING, Channel, Video
|
||||
from app.titles import normalize_title
|
||||
from app.youtube.client import YouTubeClient, best_thumbnail
|
||||
@@ -156,12 +156,21 @@ def backfill_channel_recent(db: Session, yt: YouTubeClient, channel: Channel) ->
|
||||
def backfill_channel_deep(
|
||||
db: Session, yt: YouTubeClient, channel: Channel, max_pages: int = 5
|
||||
) -> int:
|
||||
"""Continue paging older uploads from the stored cursor (bounded per run)."""
|
||||
"""Continue paging older uploads from the stored cursor (bounded per run).
|
||||
|
||||
Commits the advanced cursor after every page (not just at the end) so progress
|
||||
is durable: a restart resumes from the last stored page instead of re-fetching —
|
||||
and re-charging quota for — pages already ingested. Re-checks the shared budget
|
||||
before each page and yields once it hits the backfill reserve, so a single large
|
||||
channel can't drain the day's quota in one run even with a high max_pages; the
|
||||
paced scheduler finishes the tail on later ticks (deep_requested stays set)."""
|
||||
if channel.backfill_done or not channel.uploads_playlist_id:
|
||||
return 0
|
||||
page_token = channel.backfill_cursor
|
||||
total = 0
|
||||
for _ in range(max_pages):
|
||||
if quota.backfill_reserve_reached():
|
||||
break
|
||||
data = yt.get_playlist_items(channel.uploads_playlist_id, page_token)
|
||||
rows = [
|
||||
s
|
||||
@@ -173,11 +182,11 @@ def backfill_channel_deep(
|
||||
total += insert_stubs(db, rows)
|
||||
page_token = data.get("nextPageToken")
|
||||
channel.backfill_cursor = page_token
|
||||
channel.backfill_done = not page_token
|
||||
db.add(channel)
|
||||
db.commit()
|
||||
if not page_token:
|
||||
channel.backfill_done = True
|
||||
break
|
||||
db.add(channel)
|
||||
db.commit()
|
||||
return total
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ bounds, and whether it's a secret. The resolver returns the DB override if one i
|
||||
stored Fernet-encrypted at rest (requires TOKEN_ENCRYPTION_KEY; without it, secrets can't be
|
||||
stored in the DB and stay env-only).
|
||||
"""
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -15,6 +16,12 @@ from app import security
|
||||
from app.config import settings
|
||||
from app.models import SystemConfig
|
||||
|
||||
log = logging.getLogger("siftlode.config")
|
||||
|
||||
# Config keys whose stored ciphertext failed to decrypt — tracked so the warning is logged once per
|
||||
# key, not once per request (bounded by the number of secret config keys).
|
||||
_decrypt_warned: set[str] = set()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ConfigSpec:
|
||||
@@ -86,6 +93,7 @@ SPECS: tuple[ConfigSpec, ...] = (
|
||||
ConfigSpec("plex_server_token", "str", "plex", "plex_server_token", secret=True),
|
||||
ConfigSpec("plex_path_map", "str", "plex", "plex_path_map"),
|
||||
ConfigSpec("plex_libraries", "str", "plex", "plex_libraries"),
|
||||
ConfigSpec("plex_overlay_map", "str", "plex", "plex_overlay_map"),
|
||||
# (plex_sync_interval_min is tuned from the Scheduler dashboard, not here — it's a scheduler job.)
|
||||
ConfigSpec("plex_max_transcodes", "int", "plex", "plex_max_transcodes", min=0, max=16),
|
||||
# NOTE: plex_watch_sync_interval_min (config.py default) is surfaced here in Phase C, when the
|
||||
@@ -117,11 +125,30 @@ def _coerce(s: ConfigSpec, raw: str):
|
||||
|
||||
|
||||
def _raw_override(db: Session, key: str) -> str | None:
|
||||
"""Stored (decrypted) override string for `key`, or None if no override is set."""
|
||||
"""Stored (decrypted) override string for `key`, or None if no override is set.
|
||||
|
||||
A decrypt failure — a rotated or removed ``TOKEN_ENCRYPTION_KEY`` vs the key the value was written
|
||||
with — must NOT crash the caller. `_raw_override` sits on hot paths (`/api/me` via `google_enabled`,
|
||||
login, every YouTube scheduler job, account purge), so a raw ``InvalidToken`` here would 500 the
|
||||
whole app and the SPA wouldn't even boot. Fall back to the env/config default instead, and warn once
|
||||
per key so the operator gets a real signal without a per-request traceback. Recovery is documented in
|
||||
docs/self-hosting.md: the key must never change once secrets are stored. (B-core-2)
|
||||
"""
|
||||
row = db.get(SystemConfig, key)
|
||||
if row is None or row.value is None:
|
||||
return None
|
||||
return security.decrypt(row.value) if row.encrypted else row.value
|
||||
if not row.encrypted:
|
||||
return row.value
|
||||
try:
|
||||
return security.decrypt(row.value)
|
||||
except Exception: # noqa: BLE001 — Fernet InvalidToken / malformed ciphertext (key mismatch)
|
||||
if key not in _decrypt_warned:
|
||||
_decrypt_warned.add(key)
|
||||
log.warning(
|
||||
"config %r could not be decrypted (TOKEN_ENCRYPTION_KEY changed or missing?) — falling "
|
||||
"back to the env/config default; the key must not change once secrets are stored", key,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def get(db: Session, key: str):
|
||||
|
||||
@@ -23,3 +23,16 @@ def valid_email(value: str | None) -> bool:
|
||||
def now_utc() -> datetime:
|
||||
"""Current time as a timezone-aware UTC datetime."""
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def to_tsquery_str(q: str) -> str | None:
|
||||
"""Turn free-text into a to_tsquery() string: word runs ANDed together, the LAST word
|
||||
prefix-matched (`:*`) for as-you-type search — e.g. "ti amo magyar" → "ti & amo & magyar:*".
|
||||
Only `\\w` runs containing an alphanumeric are kept, so no tsquery operators can leak in
|
||||
(the string is built from sanitised tokens, never the raw query). Shared by the YouTube feed
|
||||
search and the Plex library search, hence this neutral home rather than either domain module."""
|
||||
tokens = [t for t in re.findall(r"\w+", q, flags=re.UNICODE) if any(c.isalnum() for c in t)]
|
||||
if not tokens:
|
||||
return None
|
||||
tokens[-1] = tokens[-1] + ":*"
|
||||
return " & ".join(tokens)
|
||||
|
||||
+73
-76
@@ -35,7 +35,8 @@ from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import errors as dlerrors
|
||||
from app.downloads import formats, quota, service, storage
|
||||
from app.downloads import adapters, formats, quota, service, storage
|
||||
from app.downloads.status import STOPPED_BY_ROUTE, JobStatus
|
||||
from app.models import DownloadJob, MediaAsset
|
||||
from app.notifications import create_notification
|
||||
from app.titles import normalize_title
|
||||
@@ -177,19 +178,36 @@ def _recover_orphans() -> None:
|
||||
def _claim_job() -> int | None:
|
||||
"""Claim the oldest queued job whose user is under their per-user concurrency limit.
|
||||
|
||||
Fetches a window of locked candidates (FOR UPDATE SKIP LOCKED) and picks the first eligible
|
||||
one, so one user flooding the queue can't starve others beyond their max_concurrent."""
|
||||
Users already AT their concurrency limit are excluded IN THE SQL, so one user flooding the
|
||||
queue can't monopolize the fetch window and starve everyone behind them. (The old fixed
|
||||
LIMIT-50 window did exactly that: 50 queued jobs from a single capped user filled the window,
|
||||
`chosen` came back None, and every other user's work — sitting past position 50 — was never
|
||||
seen.) The per-user limit is the quota row's `max_concurrent` (else the sysconfig default);
|
||||
`unlimited` users are never capped. A final Python `at_concurrency_limit` re-check still guards
|
||||
the tiny race between the exclusion subquery and the claim (it can only ever reject, never
|
||||
over-claim), so correctness never depends on the two observations agreeing."""
|
||||
with SessionLocal() as db:
|
||||
default_mc = sysconfig.get_int(db, "download_default_max_concurrent")
|
||||
rows = db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id, user_id FROM download_jobs
|
||||
WHERE status='queued'
|
||||
AND user_id NOT IN (
|
||||
SELECT r.user_id
|
||||
FROM download_jobs r
|
||||
LEFT JOIN download_quotas q ON q.user_id = r.user_id
|
||||
WHERE r.status='running' AND r.user_id IS NOT NULL
|
||||
GROUP BY r.user_id, q.max_concurrent, q.unlimited
|
||||
HAVING COALESCE(q.unlimited, false) = false
|
||||
AND count(*) >= GREATEST(COALESCE(q.max_concurrent, :dmc), 1)
|
||||
)
|
||||
ORDER BY queue_pos, id
|
||||
FOR UPDATE SKIP LOCKED
|
||||
LIMIT 50
|
||||
"""
|
||||
)
|
||||
),
|
||||
{"dmc": default_mc},
|
||||
).fetchall()
|
||||
chosen = next(
|
||||
(jid for jid, uid in rows if not quota.at_concurrency_limit(db, uid)), None
|
||||
@@ -342,7 +360,7 @@ def _run_supervised(job_id: int, asset_id: int) -> None:
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
status, stale = _job_liveness(job_id)
|
||||
if status is None or status in ("paused", "cancelled"):
|
||||
if status is None or status in STOPPED_BY_ROUTE:
|
||||
reason = "cancelled" # a route stopped it (or the row vanished)
|
||||
break
|
||||
if status == "running":
|
||||
@@ -465,31 +483,17 @@ def _finish_from_cache(job_id: int, asset_id: int) -> None:
|
||||
job = db.get(DownloadJob, job_id)
|
||||
asset = db.get(MediaAsset, asset_id)
|
||||
if job and asset:
|
||||
job.status = "done"
|
||||
job.progress = 100
|
||||
job.phase = None
|
||||
if not job.display_name:
|
||||
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None}
|
||||
if not job.display_name and asset.title:
|
||||
# display_name is VARCHAR(255); a title can be longer (e.g. a recipe in the title).
|
||||
job.display_name = asset.title[:255] if asset.title else None
|
||||
fields["display_name"] = asset.title[:255]
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
# Atomic finalize: only marks the job done while it is still running, so a concurrent
|
||||
# cancel/pause is never resurrected (see service.finalize_if_running).
|
||||
service.finalize_if_running(db, job_id, **fields)
|
||||
db.commit()
|
||||
|
||||
|
||||
def build_uploader_url(info: dict) -> str | None:
|
||||
"""The channel/uploader page URL for the auto-clickable channel line. YouTube gives a
|
||||
`channel_url`/`uploader_url` directly; Facebook gives neither but exposes a numeric
|
||||
`uploader_id` (the page id) that resolves at facebook.com/<id>. Returns None for sources with
|
||||
no derivable channel page (e.g. reddit / a raw media URL)."""
|
||||
url = info.get("channel_url") or info.get("uploader_url")
|
||||
if url:
|
||||
return url
|
||||
uid = info.get("uploader_id")
|
||||
tag = f"{info.get('extractor_key') or info.get('extractor') or ''} {info.get('webpage_url') or ''}".lower()
|
||||
if uid and "facebook" in tag:
|
||||
return f"https://www.facebook.com/{uid}"
|
||||
return None
|
||||
|
||||
|
||||
def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
"""Populate an asset's display metadata from yt-dlp's info_dict the first time it's seen
|
||||
(only if not already filled at enqueue from our catalog) — so an ad-hoc URL's row shows a
|
||||
@@ -503,7 +507,7 @@ def _fill_asset_meta_early(asset_id: int, info: dict) -> None:
|
||||
# the final title with its own source_ref fallback.
|
||||
asset.title = normalize_title(info.get("title")) or info.get("id")
|
||||
asset.uploader = info.get("uploader") or info.get("channel")
|
||||
asset.uploader_url = build_uploader_url(info)
|
||||
asset.uploader_url = adapters.resolve_adapter(None, None, info).uploader_url(info)
|
||||
asset.thumbnail_url = info.get("thumbnail")
|
||||
asset.duration_s = int(info["duration"]) if info.get("duration") else None
|
||||
asset.upload_date = _parse_upload_date(info.get("upload_date"))
|
||||
@@ -713,28 +717,17 @@ def _media_info(info: dict) -> dict:
|
||||
return {**info, **chosen}
|
||||
|
||||
|
||||
# Errors worth retrying with a different player-client set (YouTube per-client flakiness).
|
||||
_RETRIABLE_MARKERS = (
|
||||
"not a bot",
|
||||
"DRM protected",
|
||||
"Requested format is not available",
|
||||
"Only images are available",
|
||||
"Sign in to confirm",
|
||||
)
|
||||
|
||||
|
||||
def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, staging: Path):
|
||||
"""Download via yt-dlp, trying the primary player clients then the POT-backed fallback set
|
||||
on a bot/DRM/format failure. Returns the info dict of the successful attempt."""
|
||||
client_sets = [
|
||||
c for c in (settings.player_client_list, settings.player_client_fallback_list) if c
|
||||
] or [None]
|
||||
def _extract_with_fallback(job_id, asset_id, url, spec, staging, adapter):
|
||||
"""Download via yt-dlp using the source `adapter`'s client sets + options. For YouTube the adapter
|
||||
supplies the primary player-client set then a POT-backed fallback (retried on a bot/DRM/format
|
||||
error); other sources make a single attempt. Returns the info dict of the successful attempt."""
|
||||
client_sets = adapter.client_sets()
|
||||
outtmpl = str(staging / "%(id)s.%(ext)s")
|
||||
last_exc: Exception | None = None
|
||||
for i, clients in enumerate(client_sets):
|
||||
opts = formats.build_ydl_opts(
|
||||
spec, outtmpl, _make_progress_hook(job_id, asset_id),
|
||||
settings.download_pot_base_url, clients,
|
||||
extra=adapter.ydl_opts_extra(clients),
|
||||
)
|
||||
opts["postprocessor_hooks"] = [_make_pp_hook(job_id)]
|
||||
_touch_job(job_id) # stall-watchdog heartbeat: extraction writes no progress, so a slow /
|
||||
@@ -746,8 +739,7 @@ def _extract_with_fallback(job_id: int, asset_id: int, url: str, spec: dict, sta
|
||||
return ydl.extract_info(url, download=True)
|
||||
except yt_dlp.utils.DownloadError as exc:
|
||||
last_exc = exc
|
||||
retriable = any(m in str(exc) for m in _RETRIABLE_MARKERS)
|
||||
if i < len(client_sets) - 1 and retriable:
|
||||
if i < len(client_sets) - 1 and adapter.retriable(str(exc)):
|
||||
log.info("job %s: clients %s failed (%s); retrying with fallback", job_id, clients, str(exc)[:70])
|
||||
for p in list(staging.iterdir()): # drop partials before the retry
|
||||
try:
|
||||
@@ -815,8 +807,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
staging.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
_set_job(job_id, phase="downloading", progress=0)
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref)
|
||||
url = service.source_url(source_kind, source_ref)
|
||||
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging))
|
||||
info = _media_info(_extract_with_fallback(job_id, asset_id, url, spec, staging, adapter))
|
||||
# Re-resolve now that yt-dlp's info reveals the true source (a bare/redirected url only
|
||||
# identifies post-extraction) so the .nfo uniqueid type + uploader_url agree on one source.
|
||||
adapter = adapters.resolve_adapter(source_kind, source_ref, info)
|
||||
|
||||
produced = _produced_file(info, staging)
|
||||
produced = _ensure_browser_playable(job_id, produced, spec, info)
|
||||
@@ -826,6 +822,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
title=normalize_title(info.get("title")) or source_ref,
|
||||
uploader=info.get("uploader") or info.get("channel") or "Unknown Channel",
|
||||
upload_date=_parse_upload_date(info.get("upload_date")),
|
||||
uniqueid_type=adapter.nfo_uniqueid_type(),
|
||||
duration_s=int(info["duration"]) if info.get("duration") else None,
|
||||
description=info.get("description"),
|
||||
thumbnail_url=info.get("thumbnail"),
|
||||
@@ -859,7 +856,7 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
asset.acodec = info.get("acodec") if info.get("acodec") not in (None, "none") else None
|
||||
asset.title = meta.title
|
||||
asset.uploader = meta.uploader
|
||||
asset.uploader_url = build_uploader_url(info)
|
||||
asset.uploader_url = adapter.uploader_url(info)
|
||||
asset.upload_date = meta.upload_date
|
||||
asset.thumbnail_url = meta.thumbnail_url
|
||||
# Record a self-hosted poster for every download: it's the reliable link-preview
|
||||
@@ -873,20 +870,20 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = expires
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job:
|
||||
job.status = "done"
|
||||
job.progress = 100
|
||||
job.phase = None
|
||||
job.speed_bps = None
|
||||
job.eta_s = None
|
||||
if not job.display_name:
|
||||
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
||||
job.display_name = meta.title[:255] if meta.title else None
|
||||
# Capture the owner before commit expires the row (avoids a refresh SELECT just to notify).
|
||||
owner_id = job.user_id if job else None
|
||||
fields = {"status": JobStatus.DONE, "progress": 100, "phase": None, "speed_bps": None,
|
||||
"eta_s": None}
|
||||
if job and not job.display_name and meta.title:
|
||||
# display_name is VARCHAR(255) — clip a long title (see _fill_asset_meta_early).
|
||||
fields["display_name"] = meta.title[:255]
|
||||
# Atomic terminal write: finalizes ONLY while the row is still running, so a child that
|
||||
# finishes just after a route cancel/pause can't resurrect it (see finalize_if_running).
|
||||
finalized = service.finalize_if_running(db, job_id, **fields)
|
||||
db.commit()
|
||||
_notify_download_done(db, owner_id, meta.title, job_id, kind="download")
|
||||
log.info("job %s done: %s", job_id, rel)
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, meta.title, job_id, kind="download")
|
||||
log.info("job %s done: %s", job_id, rel)
|
||||
|
||||
except (_Aborted, editmod.EditAborted):
|
||||
# Paused/canceled mid-download or mid-transcode: free the asset for a later retry; keep the
|
||||
@@ -900,12 +897,12 @@ def _download(job_id: int, asset_id: int, source_kind: str, source_ref: str, spe
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job:
|
||||
job.status = "error"
|
||||
job.error = msg
|
||||
job.error_code = dlerrors.classify(msg)
|
||||
job.phase = None
|
||||
# Atomic fail: only while still running, so a route-set cancel/pause is never overwritten
|
||||
# with `error` (error also holds the asset → would re-open the double-release class).
|
||||
service.finalize_if_running(
|
||||
db, job_id, status=JobStatus.ERROR, error=msg,
|
||||
error_code=dlerrors.classify(msg), phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("job %s failed: %s", job_id, msg)
|
||||
finally:
|
||||
@@ -932,12 +929,11 @@ def _fail_edit(job_id: int, asset_id: int, msg: str) -> None:
|
||||
if asset:
|
||||
asset.status = "error"
|
||||
asset.error = msg
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job:
|
||||
job.status = "error"
|
||||
job.error = msg
|
||||
job.error_code = dlerrors.classify(msg)
|
||||
job.phase = None
|
||||
# Atomic fail: only while still running (see the download error path).
|
||||
service.finalize_if_running(
|
||||
db, job_id, status=JobStatus.ERROR, error=msg,
|
||||
error_code=dlerrors.classify(msg), phase=None,
|
||||
)
|
||||
db.commit()
|
||||
log.warning("edit job %s failed: %s", job_id, msg)
|
||||
|
||||
@@ -1013,16 +1009,17 @@ def _process_edit(job_id: int, asset_id: int) -> None:
|
||||
asset.last_access_at = datetime.now(timezone.utc)
|
||||
asset.expires_at = expires
|
||||
job = db.get(DownloadJob, job_id)
|
||||
if job:
|
||||
job.status = "done"
|
||||
job.progress = 100
|
||||
job.phase = None
|
||||
# Capture owner + title before commit expires the rows (avoids a refresh SELECT).
|
||||
owner_id = job.user_id if job else None
|
||||
dl_title = asset.title if asset else None
|
||||
# Atomic terminal write (see the download path): don't resurrect a route-stopped job.
|
||||
finalized = service.finalize_if_running(
|
||||
db, job_id, status=JobStatus.DONE, progress=100, phase=None
|
||||
)
|
||||
db.commit()
|
||||
_notify_download_done(db, owner_id, dl_title, job_id, kind="edit")
|
||||
log.info("edit job %s done: %s", job_id, rel)
|
||||
if finalized:
|
||||
_notify_download_done(db, owner_id, dl_title, job_id, kind="edit")
|
||||
log.info("edit job %s done: %s", job_id, rel)
|
||||
|
||||
except editmod.EditAborted:
|
||||
# Paused/canceled mid-encode: free the asset for a later retry (ref_count is the route's job).
|
||||
|
||||
@@ -79,7 +79,12 @@ class YouTubeClient:
|
||||
cached = decrypt_optional(tok.access_token)
|
||||
if cached and tok.expiry and tok.expiry > now + timedelta(seconds=60):
|
||||
return cached
|
||||
refresh = decrypt(tok.refresh_token_enc)
|
||||
try:
|
||||
refresh = decrypt(tok.refresh_token_enc)
|
||||
except Exception as exc: # noqa: BLE001 — Fernet InvalidToken (TOKEN_ENCRYPTION_KEY changed)
|
||||
# A key mismatch must degrade like any other token problem (caller handles YouTubeError),
|
||||
# not surface as a raw 500 on every scheduler job / connect check. (B-core-2)
|
||||
raise YouTubeError("Stored OAuth token is unreadable — reconnect your account") from exc
|
||||
if not refresh:
|
||||
raise YouTubeError("No refresh token; user must re-authenticate")
|
||||
resp = self._send(
|
||||
|
||||
@@ -29,6 +29,9 @@ services:
|
||||
condition: service_healthy
|
||||
environment:
|
||||
DATABASE_URL: postgresql+psycopg://siftlode:siftlode@testdb:5432/siftlode_test
|
||||
# This lane HAS a Postgres + ffmpeg, so the DB/ffmpeg tests must actually run here: conftest turns
|
||||
# a would-be silent skip into a hard failure (SIFTLODE_TEST_REQUIRE_DB).
|
||||
SIFTLODE_TEST_REQUIRE_DB: "1"
|
||||
volumes:
|
||||
- ./:/app
|
||||
working_dir: /app
|
||||
|
||||
+4
-2
@@ -1,6 +1,8 @@
|
||||
[pytest]
|
||||
# Run from backend/ (the image's /app). `pythonpath = .` puts the app package on sys.path so tests
|
||||
# import it as `app.*`, matching runtime. The suite is pure-logic only (no DB/network) — see tests/.
|
||||
# import it as `app.*`, matching runtime. Most of the suite is pure-logic; the DB-backed lane opts in
|
||||
# via the `db` fixture and runs in the compose test image. `-rs` reports skip REASONS so a silently
|
||||
# skipped lane is visible (the compose lane sets SIFTLODE_TEST_REQUIRE_DB=1 to make skips a hard error).
|
||||
testpaths = tests
|
||||
pythonpath = .
|
||||
addopts = -q
|
||||
addopts = -q -rs
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
# Pinned dependency lock for reproducible builds (SEC-5). GENERATED — do not hand-edit.
|
||||
# Source of truth is requirements.txt (the floor ranges); regenerate after editing it with:
|
||||
# docker run --rm -v "$PWD":/b python:3.12-slim sh -c 'pip install -q -r /b/requirements.txt && pip freeze --exclude pip --exclude setuptools --exclude wheel' | sort
|
||||
# The prod Dockerfile + Dockerfile.test + CI install THIS file; pip-audit scans it in CI.
|
||||
APScheduler==3.11.3
|
||||
Authlib==1.7.2
|
||||
Mako==1.4.1
|
||||
MarkupSafe==3.0.3
|
||||
PyYAML==6.0.3
|
||||
SQLAlchemy==2.0.52
|
||||
alembic==1.19.1
|
||||
annotated-doc==0.0.5
|
||||
annotated-types==0.8.0
|
||||
anyio==4.14.2
|
||||
argon2-cffi-bindings==25.1.0
|
||||
argon2-cffi==25.1.0
|
||||
bgutil-ytdlp-pot-provider==1.3.1
|
||||
brotli==1.2.0
|
||||
certifi==2026.7.22
|
||||
cffi==2.1.1
|
||||
charset-normalizer==3.5.1
|
||||
click==8.4.2
|
||||
cryptography==50.0.0
|
||||
fastapi==0.141.1
|
||||
feedparser-sgmllib==2.1.0
|
||||
feedparser==6.0.14
|
||||
greenlet==3.5.5
|
||||
h11==0.16.0
|
||||
httpcore==1.0.9
|
||||
httptools==0.8.0
|
||||
httpx==0.28.1
|
||||
idna==3.18
|
||||
itsdangerous==2.2.0
|
||||
joserfc==1.7.4
|
||||
mutagen==1.48.1
|
||||
numpy==2.5.2
|
||||
psycopg-binary==3.3.4
|
||||
psycopg==3.3.4
|
||||
py3langid==0.3.0
|
||||
pycparser==3.0
|
||||
pycryptodomex==3.23.0
|
||||
pydantic-settings==2.15.0
|
||||
pydantic==2.13.4
|
||||
pydantic_core==2.46.4
|
||||
python-dotenv==1.2.3
|
||||
requests==2.34.2
|
||||
starlette==1.6.0
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
tzlocal==5.4.4
|
||||
urllib3==2.7.0
|
||||
uvicorn==0.52.3
|
||||
uvloop==0.22.1
|
||||
watchfiles==1.2.0
|
||||
websockets==17.0.1
|
||||
yt-dlp-ejs==0.8.0
|
||||
yt-dlp==2026.7.4
|
||||
@@ -11,7 +11,7 @@ feedparser>=6.0,<7.0
|
||||
apscheduler>=3.10,<4.0
|
||||
py3langid>=0.3,<1.0
|
||||
itsdangerous>=2.1,<3.0
|
||||
cryptography>=46.0.7,<47
|
||||
cryptography>=50.0.0,<51
|
||||
argon2-cffi>=23.1,<26
|
||||
# Download center: yt-dlp does the extraction/download; ffmpeg (OS-level in the Dockerfile)
|
||||
# does the merge + postprocessing. The [default] extra pulls yt-dlp-ejs (the challenge-solver
|
||||
|
||||
+119
-1
@@ -12,14 +12,31 @@ config the videos.search_vector generated column needs). Building it via migrati
|
||||
that the migration chain applies cleanly. Each test then gets a clean database via TRUNCATE so
|
||||
tests that COMMIT (the whole point of this lane — transaction-ownership behaviour) don't leak.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
import app.models # noqa: F401 — register every model on Base.metadata
|
||||
import app.state as app_state
|
||||
from app import security
|
||||
from app.auth import current_user
|
||||
from app.config import settings
|
||||
from app.db import Base, SessionLocal, engine
|
||||
from app.db import Base, SessionLocal, engine, get_db
|
||||
from app.main import app
|
||||
from app.models import User
|
||||
|
||||
# Sentinel for "this dependency had no override before the fixture set one" (see override_dep).
|
||||
_MISSING = object()
|
||||
|
||||
# The compose test lane sets this so a would-be silent skip (no DB, or no ffmpeg) becomes a hard error
|
||||
# — the gate must not print green while the DB-backed / ffmpeg tests were quietly not run.
|
||||
_REQUIRE_FULL = os.getenv("SIFTLODE_TEST_REQUIRE_DB") == "1"
|
||||
|
||||
|
||||
def _db_reachable() -> bool:
|
||||
@@ -59,6 +76,23 @@ def _truncate_all():
|
||||
conn.execute(text(f"TRUNCATE {tables} RESTART IDENTITY CASCADE"))
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Fail the whole run — loudly, once — when SIFTLODE_TEST_REQUIRE_DB=1 but the environment can't
|
||||
actually run the full suite. Without this, an unreachable DATABASE_URL or a missing ffmpeg turns
|
||||
~60 DB tests + the poster tests into silent skips and the gate still exits green."""
|
||||
if not _REQUIRE_FULL:
|
||||
return
|
||||
if not _DB_REACHABLE:
|
||||
raise pytest.UsageError(
|
||||
"SIFTLODE_TEST_REQUIRE_DB=1 but no Postgres is reachable at the test DATABASE_URL — the "
|
||||
"DB-backed lane would silently skip. Fix the DB wiring or unset the flag."
|
||||
)
|
||||
if shutil.which("ffmpeg") is None:
|
||||
raise pytest.UsageError(
|
||||
"SIFTLODE_TEST_REQUIRE_DB=1 but ffmpeg is missing — the thumbnail/poster tests would skip."
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def db(_schema):
|
||||
"""A real session against the test Postgres, on a clean (empty) database. The code under test
|
||||
@@ -70,3 +104,87 @@ def db(_schema):
|
||||
finally:
|
||||
session.rollback()
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_user(db):
|
||||
"""Factory: create + commit a User in the test DB. Defaults produce a login-ready account (email
|
||||
verified, active, with a known password); override any of it via kwargs (e.g. `role="admin"`,
|
||||
`is_suspended=True`, `password=None` for a Google-only account). Returns the User."""
|
||||
seq = {"n": 0}
|
||||
|
||||
def _make(email: str | None = None, *, password: str | None = "correct horse battery", **kw):
|
||||
seq["n"] += 1
|
||||
kw.setdefault("email_verified", True)
|
||||
user = User(
|
||||
email=email or f"user{seq['n']}@example.com",
|
||||
password_hash=security.hash_password(password) if password else None,
|
||||
**kw,
|
||||
)
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return user
|
||||
|
||||
return _make
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def override_dep():
|
||||
"""Set app.dependency_overrides for the duration of a test and restore the PRIOR value per key on
|
||||
teardown — instead of a blanket `app.dependency_overrides.clear()`, which wipes every override and
|
||||
stops fixtures composing (one fixture's teardown clobbering another's)."""
|
||||
saved: dict = {}
|
||||
|
||||
def _set(dep, override):
|
||||
if dep not in saved: # remember the pre-fixture value the FIRST time we touch this key
|
||||
saved[dep] = app.dependency_overrides.get(dep, _MISSING)
|
||||
app.dependency_overrides[dep] = override
|
||||
|
||||
yield _set
|
||||
for dep, prev in saved.items():
|
||||
if prev is _MISSING:
|
||||
app.dependency_overrides.pop(dep, None)
|
||||
else:
|
||||
app.dependency_overrides[dep] = prev
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _configured(monkeypatch):
|
||||
"""The setup gate 503s /auth and /api until the wizard has run; the request-issuing fixtures below
|
||||
are all post-setup. Baked into client/as_user so no individual test has to remember it."""
|
||||
monkeypatch.setattr(app_state, "_configured_cache", True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(db, override_dep, _configured):
|
||||
"""A TestClient wired to the test DB that goes through the REAL auth/session stack — no
|
||||
`current_user` override — so login, the wallet gate and epoch revoke are exercised end to end.
|
||||
Starlette's SessionMiddleware round-trips the signed cookie, so a login in one call authenticates
|
||||
the next on this same instance. No lifespan (the scheduler / HLS sweep stay off)."""
|
||||
override_dep(get_db, lambda: db)
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def as_user(db, override_dep, _configured):
|
||||
"""Factory -> a TestClient authenticated as a given User (bypassing login), for tests that need an
|
||||
authed request but aren't exercising the login flow itself. Multiple users in ONE test are isolated:
|
||||
the shared `current_user` override resolves the caller from an `x-test-user` header, and each
|
||||
returned client tags its own requests with it — so `as_user(a)` and `as_user(b)` don't collide on
|
||||
the process-global override the way a plain `lambda: user` would."""
|
||||
registry: dict[int, User] = {}
|
||||
|
||||
def _resolver(request: Request) -> User:
|
||||
raw = request.headers.get("x-test-user")
|
||||
if raw is None or (uid := int(raw)) not in registry:
|
||||
raise HTTPException(status_code=401, detail="Not authenticated")
|
||||
return registry[uid]
|
||||
|
||||
override_dep(get_db, lambda: db)
|
||||
override_dep(current_user, _resolver)
|
||||
|
||||
def _as(user: User) -> TestClient:
|
||||
registry[user.id] = user
|
||||
return TestClient(app, headers={"x-test-user": str(user.id)})
|
||||
|
||||
return _as
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Per-source adapter selection + the source-specific policy each one carries. These lock the
|
||||
behaviour that used to live inline in worker.build_uploader_url / backfill._should_refetch."""
|
||||
from app.downloads.adapters import (
|
||||
FacebookAdapter,
|
||||
RedditAdapter,
|
||||
SourceAdapter,
|
||||
YouTubeAdapter,
|
||||
resolve_adapter,
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_by_source_kind_and_host_and_info():
|
||||
assert resolve_adapter("youtube", "abc123").key == "youtube"
|
||||
assert resolve_adapter("url", "https://www.youtube.com/watch?v=x").key == "youtube"
|
||||
assert resolve_adapter("url", "https://youtu.be/x").key == "youtube"
|
||||
assert resolve_adapter("url", "https://www.facebook.com/reel/123/").key == "facebook"
|
||||
assert resolve_adapter("url", "https://fb.watch/abc/").key == "facebook"
|
||||
assert resolve_adapter("url", "https://www.reddit.com/r/x/comments/y/z").key == "reddit"
|
||||
assert resolve_adapter("url", "https://example.com/video.mp4").key == "generic"
|
||||
# After extraction the info dict's extractor decides even for a bare/redirected url.
|
||||
assert resolve_adapter("url", None, {"extractor_key": "facebook"}).key == "facebook"
|
||||
assert resolve_adapter("url", None, {"extractor_key": "Reddit"}).key == "reddit"
|
||||
|
||||
|
||||
def test_host_match_is_netloc_not_a_loose_substring():
|
||||
# A path that merely CONTAINS "facebook.com" must not select the Facebook adapter.
|
||||
assert resolve_adapter("url", "https://example.com/facebook.com-mirror").key == "generic"
|
||||
|
||||
|
||||
def test_info_match_is_extractor_key_not_the_webpage_url():
|
||||
# The webpage_url path may carry a SIBLING source's name (a reddit slug about youtube); only the
|
||||
# yt-dlp extractor_key identifies the source, so this must resolve reddit, not youtube.
|
||||
info = {"extractor_key": "Reddit", "webpage_url": "https://reddit.com/r/x/comments/y/youtube_is_dead"}
|
||||
assert resolve_adapter("url", None, info).key == "reddit"
|
||||
|
||||
|
||||
def test_youtube_nocookie_host_still_gets_the_youtube_adapter():
|
||||
# An embed on youtube-nocookie.com is YouTube content and needs the POT/player-client strategy,
|
||||
# even though its host does not contain the substring "youtube.com".
|
||||
assert resolve_adapter("url", "https://www.youtube-nocookie.com/embed/abc123").key == "youtube"
|
||||
|
||||
|
||||
def test_lookalike_hosts_do_not_misresolve_but_subdomains_do():
|
||||
# Netloc-scoped match: a look-alike host that merely CONTAINS the domain string is NOT the source.
|
||||
assert resolve_adapter("url", "https://myyoutube.com/watch?v=x").key == "generic"
|
||||
assert resolve_adapter("url", "https://youtube.com.evil.example/x").key == "generic"
|
||||
assert resolve_adapter("url", "https://notreddit.com/r/x").key == "generic"
|
||||
# …but real subdomains still resolve.
|
||||
assert resolve_adapter("url", "https://m.youtube.com/watch?v=x").key == "youtube"
|
||||
assert resolve_adapter("url", "https://music.youtube.com/watch?v=x").key == "youtube"
|
||||
|
||||
|
||||
def test_webpage_url_netloc_identifies_source_when_extractor_key_absent():
|
||||
# The early-fill path passes info with no source_ref; the webpage_url netloc still resolves the
|
||||
# source even if extractor_key is missing from the progress-hook info_dict.
|
||||
assert resolve_adapter(None, None, {"webpage_url": "https://www.facebook.com/reel/123"}).key == "facebook"
|
||||
# A sibling source's name in the webpage_url PATH must not misresolve (host, not substring).
|
||||
info = {"extractor_key": "Reddit", "webpage_url": "https://www.reddit.com/r/x/comments/y/youtube"}
|
||||
assert resolve_adapter(None, None, info).key == "reddit"
|
||||
|
||||
|
||||
def test_uploader_url_youtube_uses_channel_url():
|
||||
a = YouTubeAdapter()
|
||||
assert a.uploader_url({"channel_url": "https://youtube.com/@chan"}) == "https://youtube.com/@chan"
|
||||
assert a.uploader_url({"uploader_url": "https://youtube.com/user/u"}) == "https://youtube.com/user/u"
|
||||
assert a.uploader_url({}) is None
|
||||
|
||||
|
||||
def test_uploader_url_facebook_falls_back_to_uploader_id():
|
||||
a = FacebookAdapter()
|
||||
assert a.uploader_url({"uploader_id": "12345"}) == "https://www.facebook.com/12345"
|
||||
# A direct channel_url still wins over the id fallback.
|
||||
assert a.uploader_url({"channel_url": "https://facebook.com/pageX", "uploader_id": "9"}) == (
|
||||
"https://facebook.com/pageX"
|
||||
)
|
||||
assert a.uploader_url({}) is None
|
||||
|
||||
|
||||
def test_uploader_url_reddit_and_generic_have_no_channel():
|
||||
assert RedditAdapter().uploader_url({"uploader_id": "99"}) is None # not facebook → no fallback
|
||||
assert SourceAdapter().uploader_url({}) is None
|
||||
|
||||
|
||||
def test_generic_source_url_is_passthrough_youtube_reconstructs():
|
||||
assert SourceAdapter().source_url("https://x/y") == "https://x/y"
|
||||
assert YouTubeAdapter().source_url("VIDID12345_") == "https://www.youtube.com/watch?v=VIDID12345_"
|
||||
|
||||
|
||||
def test_client_sets_and_retriable_are_youtube_only():
|
||||
# Generic sources make a single attempt and never retry on the YouTube per-client markers.
|
||||
assert SourceAdapter().client_sets() == [None]
|
||||
assert SourceAdapter().retriable("Sign in to confirm you're not a bot") is False
|
||||
assert YouTubeAdapter().retriable("Sign in to confirm you're not a bot") is True
|
||||
assert YouTubeAdapter().retriable("some unrelated network blip") is False
|
||||
|
||||
|
||||
def test_youtube_ydl_opts_extra_carries_player_client_and_pot():
|
||||
ea = YouTubeAdapter().ydl_opts_extra(["android_vr"])["extractor_args"]
|
||||
assert ea["youtube"]["player_client"] == ["android_vr"]
|
||||
assert "youtubepot-bgutilhttp" in ea # the bgutil PO-token base_url is wired
|
||||
# A client-less attempt still wires POT but no player_client key.
|
||||
assert "youtube" not in YouTubeAdapter().ydl_opts_extra(None)["extractor_args"]
|
||||
assert SourceAdapter().ydl_opts_extra(["android_vr"]) == {} # generic contributes nothing
|
||||
|
||||
|
||||
def test_wants_metadata_backfill_matches_old_should_refetch():
|
||||
assert YouTubeAdapter().wants_metadata_backfill() is True
|
||||
assert FacebookAdapter().wants_metadata_backfill() is True
|
||||
assert RedditAdapter().wants_metadata_backfill() is False
|
||||
assert SourceAdapter().wants_metadata_backfill() is False
|
||||
|
||||
|
||||
def test_nfo_uniqueid_type_is_the_adapter_key():
|
||||
assert YouTubeAdapter().nfo_uniqueid_type() == "youtube"
|
||||
assert SourceAdapter().nfo_uniqueid_type() == "generic"
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Auth / session security core (TEST-1), DB-lane.
|
||||
|
||||
The wallet impersonation gate, epoch-based session revocation, single-use tokens and the
|
||||
enumeration-safe uniform 401 are the security mag, and had zero tests (only `_oauth_redirect_uri` was
|
||||
covered). Starlette's SessionMiddleware round-trips the signed cookie under TestClient, so the whole
|
||||
browser-less flow is exercisable. Uses the real auth stack via the `client` fixture (NO current_user
|
||||
override) so login, the wallet gate and epoch revoke are the code under test.
|
||||
|
||||
Note: `/api/me` is the bootstrap probe — it returns 200 `{authenticated: False}` when logged out, not a
|
||||
401. A revoked session is therefore checked on a PROTECTED endpoint (`/api/feed`), which does 401.
|
||||
"""
|
||||
from datetime import timedelta
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.auth import ACTIVE_ACCOUNT_HEADER
|
||||
from app.main import app
|
||||
|
||||
_PW = "correct horse battery"
|
||||
|
||||
# The `client`/`as_user` fixtures bake in the post-setup `_configured` state, so no per-test patch here.
|
||||
|
||||
|
||||
def _login(client, email, password):
|
||||
return client.post("/auth/password-login", json={"email": email, "password": password})
|
||||
|
||||
|
||||
def test_password_login_then_me_is_authenticated(client, make_user):
|
||||
make_user(email="alice@example.com", password=_PW)
|
||||
assert _login(client, "alice@example.com", _PW).status_code == 200
|
||||
me = client.get("/api/me")
|
||||
assert me.status_code == 200
|
||||
assert me.json()["authenticated"] is True
|
||||
assert me.json()["email"] == "alice@example.com"
|
||||
|
||||
|
||||
def test_wrong_password_and_unknown_email_give_a_byte_identical_401(client, make_user):
|
||||
make_user(email="bob@example.com", password=_PW)
|
||||
wrong_pw = _login(client, "bob@example.com", "not the password")
|
||||
unknown = _login(client, "nobody@example.com", _PW)
|
||||
assert wrong_pw.status_code == 401 and unknown.status_code == 401
|
||||
# Identical body: a wrong password and a non-existent account must be indistinguishable (no
|
||||
# account-enumeration oracle).
|
||||
assert wrong_pw.json() == unknown.json()
|
||||
|
||||
|
||||
def test_a_non_wallet_account_header_falls_back_to_the_default(client, make_user):
|
||||
# The X-Siftlode-Account header may only re-target an account already signed into THIS browser
|
||||
# (in the session wallet). A header naming someone else must be ignored, not honored — otherwise
|
||||
# it's an impersonation gate hole.
|
||||
make_user(email="alice2@example.com", password=_PW)
|
||||
mallory = make_user(email="mallory@example.com", password=_PW)
|
||||
assert _login(client, "alice2@example.com", _PW).status_code == 200
|
||||
me = client.get("/api/me", headers={ACTIVE_ACCOUNT_HEADER: str(mallory.id)})
|
||||
assert me.status_code == 200
|
||||
assert me.json()["email"] == "alice2@example.com" # still Alice, not Mallory
|
||||
|
||||
|
||||
def test_logout_others_revokes_the_other_session_but_keeps_the_caller(client, make_user):
|
||||
# Two independent cookie jars (browsers) signed into the same account. `logout-others` from one
|
||||
# bumps the account epoch so the OTHER's cookie is rejected on its next request, while the caller
|
||||
# is re-stamped and stays in. Checked on /api/feed (a protected endpoint that 401s on revocation).
|
||||
make_user(email="carol@example.com", password=_PW)
|
||||
# A second, independent cookie jar; the `client` fixture already routed get_db to the test DB on
|
||||
# the shared app, so this instance uses it too — it just has its own session cookie.
|
||||
other = TestClient(app)
|
||||
assert _login(client, "carol@example.com", _PW).status_code == 200
|
||||
assert _login(other, "carol@example.com", _PW).status_code == 200
|
||||
assert other.get("/api/feed").status_code == 200 # both valid to start
|
||||
|
||||
assert client.post("/auth/logout-others").status_code == 200
|
||||
assert other.get("/api/feed").status_code == 401 # its epoch is now stale
|
||||
assert client.get("/api/feed").status_code == 200 # the caller stays signed in
|
||||
|
||||
|
||||
def test_reset_confirm_is_single_use_and_kills_existing_sessions(client, make_user, db):
|
||||
from app.routes.auth import _issue_token
|
||||
|
||||
user = make_user(email="dave@example.com", password=_PW)
|
||||
assert _login(client, "dave@example.com", _PW).status_code == 200
|
||||
assert client.get("/api/feed").status_code == 200 # session valid at the pre-reset epoch
|
||||
|
||||
raw = _issue_token(db, user, "reset", timedelta(hours=1))
|
||||
db.commit()
|
||||
ok = client.post("/auth/password-reset/confirm", json={"token": raw, "password": "brand new pw!"})
|
||||
assert ok.status_code == 200
|
||||
|
||||
# A reset bumps the epoch (compromise response) -> the pre-reset cookie is now rejected.
|
||||
assert client.get("/api/feed").status_code == 401
|
||||
# The token is single-use: a replay is a clean 400, not a second reset.
|
||||
replay = client.post("/auth/password-reset/confirm", json={"token": raw, "password": "another pw!!"})
|
||||
assert replay.status_code == 400
|
||||
|
||||
|
||||
def test_activating_an_unverified_account_drops_its_preset_password(db, make_user):
|
||||
# SEC-10 (account pre-emption): an attacker registers victim@x with a password before the approved
|
||||
# owner arrives. The address was never verified, so admin approval must NOT hand that pre-set
|
||||
# password a working login — it is dropped and any session it minted is revoked.
|
||||
from app.routes.admin import _activate_user_by_email
|
||||
|
||||
victim = make_user(
|
||||
email="victim@example.com", password="attacker-chosen", email_verified=False, is_active=False
|
||||
)
|
||||
epoch0 = victim.session_epoch or 0
|
||||
_activate_user_by_email(db, "victim@example.com")
|
||||
assert victim.is_active is True
|
||||
assert victim.password_hash is None
|
||||
assert (victim.session_epoch or 0) == epoch0 + 1
|
||||
|
||||
|
||||
def test_activating_a_verified_account_keeps_its_password(db, make_user):
|
||||
# A legit account whose email IS verified (e.g. a no-SMTP registration) keeps its password on
|
||||
# approval — the guard must not break the normal flow.
|
||||
from app.routes.admin import _activate_user_by_email
|
||||
|
||||
user = make_user(
|
||||
email="legit@example.com", password="my own password", email_verified=True, is_active=False
|
||||
)
|
||||
_activate_user_by_email(db, "legit@example.com")
|
||||
assert user.is_active is True
|
||||
assert user.password_hash is not None
|
||||
@@ -0,0 +1,318 @@
|
||||
"""R14 backend coverage: the channel-detail Normal/Shorts/Live counts (moved off the manager table),
|
||||
the update_channel -> immediate-backfill wiring (incl. the demo guard), and the interactive
|
||||
"get full history" guard that keeps it from racing the scheduler's cursor or draining the budget."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.routes.channels as channels_route
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
from app.sync import runner, videos
|
||||
|
||||
_PUB = datetime(2020, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def test_channel_detail_returns_normal_shorts_live_counts(db, monkeypatch):
|
||||
# published_at set so channel_detail skips the YouTube-enrich path (no quota / network in tests).
|
||||
ch = Channel(id="UC_nsl", title="N/S/L Channel", published_at=_PUB)
|
||||
db.add(ch)
|
||||
me = User(email="nsl@example.com")
|
||||
db.add(me)
|
||||
db.flush()
|
||||
# DISTINCT counts (2 shorts, 1 live) so a short<->live filter/field swap can't pass:
|
||||
# stored 5, short 2, live 1, normal = 5 - 2 - 1 = 2.
|
||||
db.add_all(
|
||||
[
|
||||
Video(id="v_norm1", channel_id=ch.id),
|
||||
Video(id="v_norm2", channel_id=ch.id),
|
||||
Video(id="v_short1", channel_id=ch.id, is_short=True),
|
||||
Video(id="v_short2", channel_id=ch.id, is_short=True),
|
||||
Video(id="v_live1", channel_id=ch.id, live_status="live"),
|
||||
]
|
||||
)
|
||||
db.commit()
|
||||
|
||||
# The request gate 503s an unconfigured instance; flip the process-level flag (auto-restored).
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: me
|
||||
try:
|
||||
resp = TestClient(app).get(f"/api/channels/{ch.id}")
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
assert body["stored_videos"] == 5
|
||||
assert body["count_short"] == 2
|
||||
assert body["count_live"] == 1
|
||||
assert body["count_normal"] == 2
|
||||
|
||||
|
||||
def _setup_channel_and_sub(db, *, email, is_demo=False):
|
||||
ch = Channel(
|
||||
id="UC_wire",
|
||||
title="Wire",
|
||||
published_at=_PUB,
|
||||
recent_synced_at=_PUB, # so update_channel skips the in-request recent backfill (no network)
|
||||
uploads_playlist_id="UU_wire",
|
||||
)
|
||||
db.add(ch)
|
||||
user = User(email=email, is_demo=is_demo)
|
||||
db.add(user)
|
||||
db.flush()
|
||||
db.add(Subscription(user_id=user.id, channel_id=ch.id, deep_requested=False))
|
||||
db.commit()
|
||||
return ch, user
|
||||
|
||||
|
||||
def _patch_deep_requested(db, monkeypatch, user):
|
||||
calls: list[tuple] = []
|
||||
# background.add_task runs after the response in TestClient; record what would be scheduled.
|
||||
monkeypatch.setattr(channels_route, "backfill_channel_now", lambda *a, **k: calls.append(a))
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
resp = TestClient(app).patch("/api/channels/UC_wire", json={"deep_requested": True})
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
return resp, calls
|
||||
|
||||
|
||||
def test_deep_request_schedules_immediate_backfill_for_non_demo(db, monkeypatch):
|
||||
ch, me = _setup_channel_and_sub(db, email="wire@example.com")
|
||||
resp, calls = _patch_deep_requested(db, monkeypatch, me)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["deep_requested"] is True
|
||||
# scheduled exactly once, with (channel_id, user_id) — the args backfill_channel_now needs.
|
||||
assert calls == [(ch.id, me.id)]
|
||||
|
||||
|
||||
def test_deep_request_does_not_schedule_immediate_backfill_for_demo(db, monkeypatch):
|
||||
# The SHARED demo account must not be able to kick immediate full backfills (budget drain).
|
||||
_ch, demo = _setup_channel_and_sub(db, email="demo@example.com", is_demo=True)
|
||||
resp, calls = _patch_deep_requested(db, monkeypatch, demo)
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert calls == [], "demo must not schedule an immediate backfill"
|
||||
# …and deep_requested is not persisted on the shared demo subscription either.
|
||||
assert resp.json()["deep_requested"] is False
|
||||
|
||||
|
||||
class TestImmediateBackfillGuard:
|
||||
"""backfill_channel_now coordinates with the paced scheduler through _backfill_in_progress (per-
|
||||
channel mutual exclusion, both directions) + _immediate_running (a cap on interactive runs so a
|
||||
burst can't drain the shared budget)."""
|
||||
|
||||
def setup_method(self):
|
||||
runner._backfill_in_progress.clear()
|
||||
runner._immediate_running = 0
|
||||
|
||||
def teardown_method(self):
|
||||
runner._backfill_in_progress.clear()
|
||||
runner._immediate_running = 0
|
||||
|
||||
@staticmethod
|
||||
def _boom_session():
|
||||
# Reached only AFTER the guard admits the channel; if the guard short-circuits (as it must
|
||||
# in these tests) this never runs, so raising proves the short-circuit.
|
||||
raise AssertionError("guard must short-circuit before opening a session")
|
||||
|
||||
def test_a_second_call_for_an_in_flight_channel_is_a_noop(self, monkeypatch):
|
||||
runner._backfill_in_progress.add("UC_busy")
|
||||
monkeypatch.setattr(runner, "SessionLocal", self._boom_session)
|
||||
runner.backfill_channel_now("UC_busy", 1) # returns immediately; _boom_session never runs
|
||||
assert runner._backfill_in_progress == {"UC_busy"} # the in-flight entry is untouched
|
||||
|
||||
def test_over_the_concurrency_cap_falls_back_to_the_scheduler(self, monkeypatch):
|
||||
runner._immediate_running = runner._MAX_CONCURRENT_IMMEDIATE
|
||||
monkeypatch.setattr(runner, "SessionLocal", self._boom_session)
|
||||
runner.backfill_channel_now("UC_new", 1) # over the cap → no-op, scheduler will handle it
|
||||
assert "UC_new" not in runner._backfill_in_progress
|
||||
|
||||
def test_scheduler_deep_pass_skips_an_in_flight_channel(self, db, monkeypatch):
|
||||
# A channel the scheduler's query WOULD pick (deep-requested, recent-synced, not done)…
|
||||
ch = Channel(
|
||||
id="UC_inflight",
|
||||
title="In-flight",
|
||||
recent_synced_at=_PUB,
|
||||
uploads_playlist_id="UU_inflight",
|
||||
backfill_done=False,
|
||||
)
|
||||
db.add(ch)
|
||||
me = User(email="svc@example.com")
|
||||
db.add(me)
|
||||
db.flush()
|
||||
db.add(Subscription(user_id=me.id, channel_id=ch.id, deep_requested=True))
|
||||
db.commit()
|
||||
|
||||
paged: list[str] = []
|
||||
monkeypatch.setattr(runner, "get_service_user", lambda _db: me)
|
||||
monkeypatch.setattr(runner.quota, "remaining_today", lambda _db: 10_000)
|
||||
monkeypatch.setattr(runner, "YouTubeClient", lambda *a, **k: _NullYt())
|
||||
monkeypatch.setattr(
|
||||
runner, "backfill_channel_deep", lambda _db, _yt, channel, _mp: paged.append(channel.id) or 0
|
||||
)
|
||||
monkeypatch.setattr(runner, "reconcile_full_history", lambda _db: 0)
|
||||
|
||||
# …is skipped by run_deep_backfill while an interactive backfill owns it (no cursor race).
|
||||
runner._backfill_in_progress.add(ch.id)
|
||||
runner.run_deep_backfill(db)
|
||||
assert paged == [], "the scheduler must not page a channel an immediate backfill is handling"
|
||||
|
||||
# …and IS paged once the interactive backfill releases it.
|
||||
runner._backfill_in_progress.discard(ch.id)
|
||||
runner.run_deep_backfill(db)
|
||||
assert paged == [ch.id]
|
||||
|
||||
|
||||
class _NullYt:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
|
||||
def _playlist_page(ids, next_token=None):
|
||||
"""A playlistItems.list response shaped for stub_from_playlist_item."""
|
||||
page = {
|
||||
"items": [
|
||||
{
|
||||
"contentDetails": {"videoId": vid, "videoPublishedAt": "2020-01-01T00:00:00Z"},
|
||||
"snippet": {"title": vid},
|
||||
}
|
||||
for vid in ids
|
||||
]
|
||||
}
|
||||
if next_token is not None:
|
||||
page["nextPageToken"] = next_token
|
||||
return page
|
||||
|
||||
|
||||
class _PagedYt:
|
||||
"""Serves a fixed page-token -> page map and records the tokens it was asked for, so a test can
|
||||
assert exactly how many pages a deep-backfill run fetched (and where it resumed from)."""
|
||||
|
||||
def __init__(self, pages):
|
||||
self.pages = pages
|
||||
self.calls: list[str | None] = []
|
||||
|
||||
def get_playlist_items(self, playlist_id, page_token=None):
|
||||
self.calls.append(page_token)
|
||||
return self.pages[page_token]
|
||||
|
||||
|
||||
class _RaisingYt:
|
||||
"""Serves pages like _PagedYt but raises when asked for `raise_on_token`, to simulate a
|
||||
network/API failure that lands BETWEEN pages of a deep backfill."""
|
||||
|
||||
def __init__(self, pages, raise_on_token):
|
||||
self.pages = pages
|
||||
self.raise_on_token = raise_on_token
|
||||
self.calls: list[str | None] = []
|
||||
|
||||
def get_playlist_items(self, playlist_id, page_token=None):
|
||||
self.calls.append(page_token)
|
||||
if page_token == self.raise_on_token:
|
||||
raise RuntimeError("simulated mid-backfill failure")
|
||||
return self.pages[page_token]
|
||||
|
||||
|
||||
class TestDeepBackfillPaging:
|
||||
"""backfill_channel_deep must commit the advanced cursor after every page (durable resume, no
|
||||
re-charging quota for pages already stored) and yield the moment the shared budget hits the
|
||||
backfill reserve — the two properties that make a high immediate max_pages safe."""
|
||||
|
||||
@staticmethod
|
||||
def _channel(db):
|
||||
ch = Channel(
|
||||
id="UC_deep",
|
||||
title="Deep",
|
||||
recent_synced_at=_PUB,
|
||||
uploads_playlist_id="UU_deep",
|
||||
backfill_done=False,
|
||||
backfill_cursor=None,
|
||||
)
|
||||
db.add(ch)
|
||||
db.commit()
|
||||
return ch
|
||||
|
||||
def test_cursor_is_committed_each_page_and_a_later_run_resumes(self, db, monkeypatch):
|
||||
monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: False) # budget ok
|
||||
ch = self._channel(db)
|
||||
yt = _PagedYt(
|
||||
{
|
||||
None: _playlist_page(["a1", "a2"], next_token="t1"),
|
||||
"t1": _playlist_page(["b1", "b2"], next_token="t2"),
|
||||
"t2": _playlist_page(["c1"], next_token=None), # last page: no nextPageToken
|
||||
}
|
||||
)
|
||||
|
||||
# First run is bounded to 2 pages: it must stop mid-catalogue with the cursor durably parked
|
||||
# on the next unfetched page and backfill_done still False.
|
||||
added = videos.backfill_channel_deep(db, yt, ch, max_pages=2)
|
||||
assert added == 4
|
||||
assert yt.calls == [None, "t1"]
|
||||
assert ch.backfill_cursor == "t2"
|
||||
assert ch.backfill_done is False
|
||||
# Durably persisted (committed per page), not just set on the in-memory object.
|
||||
db.expire(ch)
|
||||
assert ch.backfill_cursor == "t2"
|
||||
assert db.query(Video).filter(Video.channel_id == ch.id).count() == 4
|
||||
|
||||
# A later run resumes FROM the stored cursor (not from the start — no wasted re-fetch) and
|
||||
# finishes: reaching the token-less last page flips backfill_done and clears the cursor.
|
||||
added2 = videos.backfill_channel_deep(db, yt, ch, max_pages=5)
|
||||
assert added2 == 1
|
||||
assert yt.calls == [None, "t1", "t2"] # resumed at "t2", did not re-fetch None/"t1"
|
||||
assert ch.backfill_done is True
|
||||
assert ch.backfill_cursor is None
|
||||
|
||||
def test_yields_when_the_shared_budget_hits_the_reserve(self, db, monkeypatch):
|
||||
# Budget is fine for the first page, then hits the reserve — the guard is checked BEFORE each
|
||||
# page, so the run fetches exactly one page and then yields, leaving the tail for the paced
|
||||
# scheduler (cursor parked, backfill_done False).
|
||||
reached = iter([False, True])
|
||||
monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: next(reached))
|
||||
ch = self._channel(db)
|
||||
yt = _PagedYt(
|
||||
{
|
||||
None: _playlist_page(["a1", "a2"], next_token="t1"),
|
||||
"t1": _playlist_page(["b1"], next_token="t2"),
|
||||
}
|
||||
)
|
||||
|
||||
added = videos.backfill_channel_deep(db, yt, ch, max_pages=50)
|
||||
assert added == 2
|
||||
assert yt.calls == [None] # stopped before fetching the second page
|
||||
assert ch.backfill_cursor == "t1"
|
||||
assert ch.backfill_done is False
|
||||
|
||||
def test_a_mid_run_failure_leaves_the_prior_page_durably_committed(self, db, monkeypatch):
|
||||
# The whole point of committing per page (vs one commit after the loop): if paging blows up
|
||||
# mid-run, the pages already ingested stay durable, so the next run resumes from the stored
|
||||
# cursor instead of re-fetching — and re-charging quota for — them. A single end-of-loop
|
||||
# commit would lose the advanced cursor on the crash (it never reaches the post-loop commit),
|
||||
# so this asserts the property the per-page commit actually buys.
|
||||
monkeypatch.setattr(videos.quota, "backfill_reserve_reached", lambda: False) # budget ok
|
||||
ch = self._channel(db)
|
||||
yt = _RaisingYt(
|
||||
{None: _playlist_page(["a1", "a2"], next_token="t1")},
|
||||
raise_on_token="t1", # page 1 succeeds; fetching page 2 (token 't1') fails
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
videos.backfill_channel_deep(db, yt, ch, max_pages=5)
|
||||
|
||||
# Page 1's cursor AND rows were committed before the page-2 crash — re-read from the DB
|
||||
# (expire) to prove it's persisted, not just set on the in-memory object.
|
||||
db.expire(ch)
|
||||
assert ch.backfill_cursor == "t1"
|
||||
assert ch.backfill_done is False
|
||||
assert db.query(Video).filter(Video.channel_id == ch.id).count() == 2
|
||||
@@ -0,0 +1,133 @@
|
||||
"""DB-backed tests for R12 · S5 backend residue — the two download limits:
|
||||
|
||||
- **#2 claim fairness** (`worker._claim_job`): users already AT their per-user concurrency limit
|
||||
are excluded IN THE SQL, so one user flooding the queue past the fetch window can't starve
|
||||
everyone behind them. The old fixed `LIMIT 50` window did exactly that.
|
||||
- **#3 list cap** (`routes.downloads.list_downloads` / `shared_with_me`): the per-user listings
|
||||
are bounded so a heavy library can't return an unbounded result set.
|
||||
|
||||
Uses the `db` fixture (real Postgres) — skipped when no database is reachable. `_claim_job` runs in
|
||||
its OWN session, so every test COMMITS its setup before calling it.
|
||||
"""
|
||||
from app.models import DownloadJob, DownloadQuota, User
|
||||
from app.worker import _claim_job
|
||||
|
||||
|
||||
def _job(db, user_id, queue_pos, status="queued"):
|
||||
j = DownloadJob(
|
||||
user_id=user_id,
|
||||
source_kind="youtube",
|
||||
source_ref="x" * 11,
|
||||
status=status,
|
||||
queue_pos=queue_pos,
|
||||
profile_snapshot={},
|
||||
)
|
||||
db.add(j)
|
||||
return j
|
||||
|
||||
|
||||
def _quota(db, user_id, max_concurrent, unlimited=False):
|
||||
db.add(
|
||||
DownloadQuota(
|
||||
user_id=user_id,
|
||||
max_bytes=10**12,
|
||||
max_concurrent=max_concurrent,
|
||||
max_jobs=10**6,
|
||||
unlimited=unlimited,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --- #2 · claim fairness --------------------------------------------------------------------
|
||||
|
||||
def test_capped_user_flood_does_not_starve_users_behind_the_window(db):
|
||||
a = User(email="flooder@example.com")
|
||||
b = User(email="waiter@example.com")
|
||||
db.add_all([a, b])
|
||||
db.flush()
|
||||
_quota(db, a.id, max_concurrent=1) # A: 1 concurrent, already running one -> all A's queued are ineligible
|
||||
_quota(db, b.id, max_concurrent=3)
|
||||
_job(db, a.id, queue_pos=0, status="running")
|
||||
for i in range(1, 61): # A floods the FRONT with far more than the 50-row fetch window
|
||||
_job(db, a.id, queue_pos=i)
|
||||
bjob = _job(db, b.id, queue_pos=100) # B sits behind the flood
|
||||
db.commit()
|
||||
|
||||
assert _claim_job() == bjob.id # B is served despite A's 60-job flood filling the old window
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, bjob.id).status == "running"
|
||||
|
||||
|
||||
def test_eligible_user_claims_oldest_by_queue_pos(db):
|
||||
u = User(email="solo@example.com")
|
||||
db.add(u)
|
||||
db.flush()
|
||||
_quota(db, u.id, max_concurrent=2)
|
||||
first = _job(db, u.id, queue_pos=1)
|
||||
_job(db, u.id, queue_pos=2)
|
||||
db.commit()
|
||||
assert _claim_job() == first.id
|
||||
|
||||
|
||||
def test_all_candidates_capped_returns_none(db):
|
||||
u = User(email="capped@example.com")
|
||||
db.add(u)
|
||||
db.flush()
|
||||
_quota(db, u.id, max_concurrent=1)
|
||||
_job(db, u.id, queue_pos=0, status="running") # at the cap
|
||||
_job(db, u.id, queue_pos=1) # queued but ineligible
|
||||
db.commit()
|
||||
assert _claim_job() is None
|
||||
|
||||
|
||||
def test_stored_zero_concurrency_is_floored_not_starving(db):
|
||||
# A stray max_concurrent=0 is only reachable by a direct DB edit (admin_set_quota floors at 1),
|
||||
# but it must not wedge the claim loop. The effective limit floors to 1 on BOTH the read path
|
||||
# (resolve -> at_concurrency_limit) and in the claim SQL (GREATEST(..., 1)), so the two "at
|
||||
# limit" checks agree: the user's own first job is claimable and its flood never monopolizes
|
||||
# the window with permanently-ineligible jobs. (Before the floor, _claim_job returned None here
|
||||
# — the guard rejected all 60 of A's jobs — and any user behind them starved.)
|
||||
a = User(email="zero@example.com")
|
||||
b = User(email="behind@example.com")
|
||||
db.add_all([a, b])
|
||||
db.flush()
|
||||
_quota(db, a.id, max_concurrent=0) # nonsensical stored value
|
||||
_quota(db, b.id, max_concurrent=2)
|
||||
for i in range(60):
|
||||
_job(db, a.id, queue_pos=i)
|
||||
_job(db, b.id, queue_pos=100)
|
||||
db.commit()
|
||||
|
||||
claimed = _claim_job()
|
||||
assert claimed is not None # the loop is not wedged
|
||||
db.expire_all()
|
||||
assert db.get(DownloadJob, claimed).user_id == a.id # A's own first job runs (floored to 1)
|
||||
|
||||
|
||||
def test_unlimited_user_is_never_excluded(db):
|
||||
u = User(email="unlimited@example.com")
|
||||
db.add(u)
|
||||
db.flush()
|
||||
_quota(db, u.id, max_concurrent=1, unlimited=True)
|
||||
for i in range(3): # 3 running, well past max_concurrent — but unlimited bypasses the cap
|
||||
_job(db, u.id, queue_pos=i, status="running")
|
||||
q = _job(db, u.id, queue_pos=10)
|
||||
db.commit()
|
||||
assert _claim_job() == q.id
|
||||
|
||||
|
||||
# --- #3 · list cap --------------------------------------------------------------------------
|
||||
|
||||
def test_list_downloads_is_capped(db, monkeypatch):
|
||||
from app.routes import downloads as dl
|
||||
|
||||
u = User(email="heavy@example.com")
|
||||
db.add(u)
|
||||
db.flush()
|
||||
for i in range(3):
|
||||
_job(db, u.id, queue_pos=i, status="done")
|
||||
db.commit()
|
||||
|
||||
monkeypatch.setattr(dl, "_MAX_LIST", 2)
|
||||
rows = dl.list_downloads(user=u, db=db)
|
||||
assert len(rows) == 2 # bounded by the cap, not the 3 rows on disk
|
||||
@@ -0,0 +1,75 @@
|
||||
"""DB-backed regression tests for `service.finalize_if_running` (B-sub-1).
|
||||
|
||||
The worker's terminal writes go through this helper so a child that finishes just after a route
|
||||
cancel/pause can NEVER flip the job back to done/error — which would re-open the double-release ->
|
||||
file-delete-under-live-holder class (done/error hold the asset, canceled does not). The guard is an
|
||||
atomic `UPDATE ... WHERE status='running'`, so these tests assert it fires exactly when the job is
|
||||
still running and is a no-op otherwise. Uses the real-Postgres `db` fixture; skipped without a DB.
|
||||
"""
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.downloads import service
|
||||
from app.downloads.status import JobStatus
|
||||
from app.models import DownloadJob, User
|
||||
|
||||
|
||||
def _running_job(db) -> DownloadJob:
|
||||
user = User(email="dl@example.com")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
job = DownloadJob(
|
||||
user_id=user.id,
|
||||
source_kind="youtube",
|
||||
source_ref="x" * 11,
|
||||
status=JobStatus.RUNNING.value,
|
||||
queue_pos=1,
|
||||
profile_snapshot={},
|
||||
)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
return job
|
||||
|
||||
|
||||
def _row(db, job_id):
|
||||
# Read straight from the DB: SessionLocal uses expire_on_commit=False, so the ORM identity-map
|
||||
# copy is stale after finalize_if_running's raw UPDATE + commit.
|
||||
return db.execute(
|
||||
text("SELECT status, progress FROM download_jobs WHERE id = :id"), {"id": job_id}
|
||||
).one()
|
||||
|
||||
|
||||
def test_finalize_marks_a_running_job_done_and_reports_it(db):
|
||||
job = _running_job(db)
|
||||
won = service.finalize_if_running(db, job.id, status=JobStatus.DONE, progress=100, phase=None)
|
||||
db.commit()
|
||||
assert won is True
|
||||
status, progress = _row(db, job.id)
|
||||
assert status == "done"
|
||||
assert progress == 100
|
||||
|
||||
|
||||
def test_finalize_is_a_noop_on_a_canceled_job(db):
|
||||
# The exact B-sub-1 scenario: a route already canceled the job (released its asset once), and a
|
||||
# finishing child then tries to write `done`. The guard must NOT resurrect it, or a later delete
|
||||
# would double-release the asset.
|
||||
job = _running_job(db)
|
||||
db.execute(
|
||||
text("UPDATE download_jobs SET status = 'canceled' WHERE id = :id"), {"id": job.id}
|
||||
)
|
||||
db.commit()
|
||||
won = service.finalize_if_running(db, job.id, status=JobStatus.DONE, progress=100)
|
||||
db.commit()
|
||||
assert won is False
|
||||
assert _row(db, job.id).status == "canceled"
|
||||
|
||||
|
||||
def test_finalize_is_a_noop_on_a_paused_job(db):
|
||||
job = _running_job(db)
|
||||
db.execute(
|
||||
text("UPDATE download_jobs SET status = 'paused' WHERE id = :id"), {"id": job.id}
|
||||
)
|
||||
db.commit()
|
||||
won = service.finalize_if_running(db, job.id, status=JobStatus.ERROR, error="boom", phase=None)
|
||||
db.commit()
|
||||
assert won is False
|
||||
assert _row(db, job.id).status == "paused"
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Feed cursor endpoint: a malformed ?cursor= must be a clean HTTP 400, not a 500.
|
||||
|
||||
Locks the refactor that split cursor handling across modules — feedquery.decode_cursor raises a
|
||||
plain ValueError and routes/feed.get_feed translates it to 400 — so a future regression (a dropped
|
||||
try/except, or a new decode_cursor caller without its own catch) turns the 400 back into a 500 and
|
||||
fails right here. DB lane (`db` fixture) — skipped without Postgres.
|
||||
"""
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import User
|
||||
|
||||
|
||||
def _get(db, user, monkeypatch, path):
|
||||
monkeypatch.setattr(state, "_configured_cache", True) # the request gate 503s an unconfigured app
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
client = TestClient(app) # no context manager: skip lifespan (scheduler/HLS sweep)
|
||||
return client.get(path)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_malformed_feed_cursor_returns_400(db, monkeypatch):
|
||||
user = User(email="feedcursor@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
resp = _get(db, user, monkeypatch, "/api/feed?cursor=not-a-real-cursor")
|
||||
assert resp.status_code == 400
|
||||
assert resp.json()["detail"] == "Invalid feed cursor"
|
||||
|
||||
|
||||
def test_feed_without_cursor_is_200(db, monkeypatch):
|
||||
# Sanity: the same endpoint without a cursor is a normal 200 (proves the 400 above is the
|
||||
# cursor path, not a globally-broken request). A fresh user has no subscriptions → empty feed.
|
||||
user = User(email="feednocursor@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
resp = _get(db, user, monkeypatch, "/api/feed")
|
||||
assert resp.status_code == 200
|
||||
assert resp.json()["items"] == []
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Feed keyset paging (TEST-2), DB-lane.
|
||||
|
||||
The feed's cursor paging (feedquery.sort_keys / order_by / keyset_predicate / encode_cursor /
|
||||
decode_cursor) is the NULLS-LAST + tiebreaker algebra where a page boundary bug hides as a duplicated or
|
||||
skipped row — on the busiest endpoint, while the scheduler ingests mid-scroll. It had zero tests.
|
||||
|
||||
The invariant here is strong and order-agnostic: for EVERY sort, walking the feed in pages of 2 via the
|
||||
cursor must reproduce EXACTLY the order a single unpaged query returns — same rows, same order, no dup, no
|
||||
skip. The seed is built to stress it: duplicate sort values (so the id tiebreaker is load-bearing) and
|
||||
NULLs in published_at / view_count / duration_seconds / title / subscriber_count (so nulls_last is
|
||||
exercised on both a first and a second key). (The malformed-cursor -> 400 path is covered by
|
||||
test_feed_cursor.py.)
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
|
||||
# Every sort feedquery.sort_keys defines except "relevance" (which needs a search query to rank).
|
||||
_SORTS = [
|
||||
"newest", "oldest", "views", "views_asc", "duration_desc", "duration_asc",
|
||||
"title", "title_desc", "subscribers", "subscribers_asc", "priority", "priority_asc", "shuffle",
|
||||
]
|
||||
|
||||
|
||||
def _dt(day: int) -> datetime:
|
||||
return datetime(2024, 1, day, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _seed(db):
|
||||
# Two channels; one has a NULL subscriber_count (exercises the subscribers sort's nulls_last).
|
||||
db.add_all([
|
||||
Channel(id="UCa", title="Channel A", subscriber_count=1000),
|
||||
Channel(id="UCb", title="Channel B", subscriber_count=None),
|
||||
])
|
||||
user = User(email="keyset@example.com", email_verified=True)
|
||||
db.add(user)
|
||||
db.flush()
|
||||
db.add_all([
|
||||
Subscription(user_id=user.id, channel_id="UCa", priority=5, hidden=False),
|
||||
Subscription(user_id=user.id, channel_id="UCb", priority=0, hidden=False),
|
||||
])
|
||||
# 7 normal/organic videos. Deliberate duplicates (published_at day 1 x2 and day 3 x2, view_count
|
||||
# 100 x2, duration 60 x2 / 30 x2) and NULLs spread across the sort columns.
|
||||
rows = [
|
||||
# id ch published views dur title
|
||||
("vid00", "UCa", _dt(1), 100, 60, "alpha"),
|
||||
("vid01", "UCa", _dt(1), 100, 60, "beta"),
|
||||
("vid02", "UCa", _dt(2), None, None, "gamma"),
|
||||
("vid03", "UCb", None, 50, 120, None),
|
||||
("vid04", "UCb", _dt(3), 200, 30, "delta"),
|
||||
("vid05", "UCb", _dt(3), None, 30, "epsilon"),
|
||||
("vid06", "UCa", _dt(28), None, None, "zeta"),
|
||||
]
|
||||
db.add_all([
|
||||
Video(
|
||||
id=vid, channel_id=ch, published_at=pub, view_count=views, duration_seconds=dur,
|
||||
title=title, is_short=False, live_status="none", via_search=False, via_explore=False,
|
||||
)
|
||||
for vid, ch, pub, views, dur, title in rows
|
||||
])
|
||||
db.commit()
|
||||
return user, [r[0] for r in rows]
|
||||
|
||||
|
||||
def _page(client, sort, cursor):
|
||||
url = f"/api/feed?sort={sort}&limit=2"
|
||||
if cursor:
|
||||
url += f"&cursor={cursor}"
|
||||
resp = client.get(url)
|
||||
assert resp.status_code == 200, resp.text
|
||||
return resp.json()
|
||||
|
||||
|
||||
def test_feed_surfaces_the_whole_seed(db, as_user):
|
||||
user, ids = _seed(db)
|
||||
body = as_user(user).get("/api/feed?limit=50").json()
|
||||
assert {it["id"] for it in body["items"]} == set(ids)
|
||||
assert body["has_more"] is False and body["next_cursor"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sort", _SORTS)
|
||||
def test_cursor_walk_matches_the_unpaged_order(db, as_user, sort):
|
||||
user, ids = _seed(db)
|
||||
client = as_user(user)
|
||||
|
||||
# Ground truth: the full order in one query (no cursor).
|
||||
full = [it["id"] for it in client.get(f"/api/feed?sort={sort}&limit=50").json()["items"]]
|
||||
# Guard against a vacuous pass: if seeding ever stops surfacing the videos, `walked == full` would
|
||||
# be [] == [] for every sort and never exercise the cursor. All 7 rows must be present.
|
||||
assert len(full) == len(ids), f"{sort}: seed did not surface all rows — the walk would be vacuous"
|
||||
|
||||
# Walk it two at a time via the cursor and reassemble.
|
||||
walked: list[str] = []
|
||||
cursor = None
|
||||
for _ in range(len(full) + 2): # generous upper bound; the loop breaks on the last page
|
||||
body = _page(client, sort, cursor)
|
||||
walked.extend(it["id"] for it in body["items"])
|
||||
cursor = body["next_cursor"]
|
||||
if not body["has_more"]:
|
||||
assert cursor is None
|
||||
break
|
||||
|
||||
assert walked == full, f"{sort}: paged order diverged from the unpaged query"
|
||||
assert len(walked) == len(set(walked)), f"{sort}: cursor paging duplicated a row"
|
||||
@@ -9,6 +9,15 @@ def _opts(spec):
|
||||
return formats.build_ydl_opts(spec, "/tmp/%(id)s.%(ext)s", lambda d: None)
|
||||
|
||||
|
||||
def test_adapter_extra_opts_are_merged_in():
|
||||
# The SourceAdapter contributes source-specific opts (e.g. YouTube's extractor_args) via `extra`.
|
||||
extra = {"extractor_args": {"youtube": {"player_client": ["web"]}}}
|
||||
opts = formats.build_ydl_opts({}, "/tmp/%(id)s.%(ext)s", lambda d: None, extra=extra)
|
||||
assert opts["extractor_args"]["youtube"]["player_client"] == ["web"]
|
||||
# No extra → no extractor_args (a generic source contributes nothing).
|
||||
assert "extractor_args" not in formats.build_ydl_opts({}, "/tmp/x", lambda d: None)
|
||||
|
||||
|
||||
def test_mp4_default_selects_avc1_with_progressive_fallback():
|
||||
fmt = _opts({"mode": "av", "container": "mp4"})["format"]
|
||||
# avc1 video-only first, then the avc1 progressive, then an unrestricted fallback.
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Migration hygiene (TEST-3), DB-lane.
|
||||
|
||||
Two things the prod boot never checks:
|
||||
- **model <-> schema drift:** `entrypoint.sh` only ever runs `alembic upgrade head`; nothing verifies the
|
||||
migrated schema still matches `Base.metadata`. A model column added without a migration (or a migration
|
||||
that drifted from the model) ships silently.
|
||||
- **the downgrade path:** every migration defines `downgrade()`, but it never runs, so a broken downgrade
|
||||
is only discovered during an emergency rollback.
|
||||
|
||||
Both use the `db` fixture, which has already brought the schema to head via `_schema`.
|
||||
|
||||
Scope note: we gate on COLUMN/TABLE drift — a table or column present on one side and not the other, or
|
||||
a column whose type/nullability disagrees. `compare_metadata` also reports index/constraint/sequence
|
||||
differences, but those are dominated by migration-only objects (partial/expression/GIN indexes, CHECK
|
||||
constraints, SERIAL sequences) that can't be expressed on `Base.metadata`, so comparing them wholesale is
|
||||
noise; they're filtered out. The column/table signal is the "someone changed a model without a migration"
|
||||
case worth failing on.
|
||||
"""
|
||||
from alembic import command
|
||||
from alembic.autogenerate import compare_metadata
|
||||
from alembic.config import Config
|
||||
from alembic.migration import MigrationContext
|
||||
|
||||
from app.db import Base, engine
|
||||
|
||||
# The diff ops that mean a real table/column disagreement (presence AND shape) — not the index/
|
||||
# constraint/sequence noise from migration-only objects.
|
||||
_DRIFT_OPS = {
|
||||
"add_table", "remove_table", "add_column", "remove_column", "modify_type", "modify_nullable",
|
||||
}
|
||||
|
||||
|
||||
def _structural_drift():
|
||||
"""Table/column presence diffs between the ORM models and the migrated schema (`compare_type=True`
|
||||
mirrors alembic/env.py). compare_metadata nests per-column ops in a sub-list per table and returns
|
||||
table-level ops as bare tuples; flatten both and keep only the presence ops."""
|
||||
with engine.connect() as conn:
|
||||
ctx = MigrationContext.configure(conn, opts={"compare_type": True})
|
||||
diffs = compare_metadata(ctx, Base.metadata)
|
||||
drift = []
|
||||
for d in diffs:
|
||||
for entry in (d if isinstance(d, list) else [d]):
|
||||
if isinstance(entry, tuple) and entry and entry[0] in _DRIFT_OPS:
|
||||
drift.append(entry)
|
||||
return drift
|
||||
|
||||
|
||||
def test_models_match_the_migrated_schema(db):
|
||||
# Non-empty == the ORM models and the alembic-built schema disagree on a table/column — almost
|
||||
# always a model change that forgot its migration.
|
||||
drift = _structural_drift()
|
||||
assert drift == [], "model/schema drift — add or fix a migration:\n" + "\n".join(
|
||||
str(d) for d in drift
|
||||
)
|
||||
|
||||
|
||||
def test_head_migration_downgrades_and_reupgrades(db):
|
||||
# Exercise the newest migration's downgrade + re-upgrade so a broken downgrade is caught the sprint
|
||||
# it lands, not during a rollback. Leaves the schema back at head for the rest of the session.
|
||||
cfg = Config("alembic.ini")
|
||||
command.downgrade(cfg, "-1")
|
||||
reupgraded = False
|
||||
try:
|
||||
command.upgrade(cfg, "head")
|
||||
reupgraded = True
|
||||
assert _structural_drift() == []
|
||||
finally:
|
||||
# If the re-upgrade itself failed, restore the SHARED schema to head so the failure doesn't
|
||||
# cascade into every later DB-lane test. Skip it when the upgrade already succeeded — re-running
|
||||
# it would be a redundant no-op and could mask a failing assert with the finally's own error.
|
||||
if not reupgraded:
|
||||
command.upgrade(cfg, "head")
|
||||
@@ -0,0 +1,75 @@
|
||||
"""create_notification title-clamp regression.
|
||||
|
||||
A downloaded FB/IG reel's "title" is the whole caption (hundreds to ~1900 chars). Before the
|
||||
clamp that string went straight into Notification.title (String(255)) and, at the batched
|
||||
download_gc commit, raised psycopg StringDataRightTruncation — aborting the entire GC run so
|
||||
no expiry deletion / LRU eviction / orphan reap ever ran. The clamp trims the title to the
|
||||
column limit centrally, defending all callers regardless of DB backend."""
|
||||
from app.models import Notification, User
|
||||
from app.notifications import TITLE_MAX_LEN, create_notification
|
||||
|
||||
|
||||
class _CaptureDB:
|
||||
"""Captures the added row without a real DB, so the clamp is verified on every lane
|
||||
(the truncation only reproduces on Postgres; the clamp logic is backend-agnostic)."""
|
||||
|
||||
def __init__(self):
|
||||
self.added: list[Notification] = []
|
||||
|
||||
def add(self, obj):
|
||||
self.added.append(obj)
|
||||
|
||||
def commit(self):
|
||||
pass
|
||||
|
||||
def refresh(self, _obj):
|
||||
pass
|
||||
|
||||
|
||||
def test_title_clamped_to_column_limit():
|
||||
long_title = "x" * 2000 # e.g. a reel caption
|
||||
notif = create_notification(
|
||||
_CaptureDB(), user_id=1, type="download_expiring", title=long_title
|
||||
)
|
||||
assert len(notif.title) == TITLE_MAX_LEN
|
||||
assert notif.title.endswith("…")
|
||||
# The untruncated title still reaches the client via `data`, which is never clamped.
|
||||
notif2 = create_notification(
|
||||
_CaptureDB(),
|
||||
user_id=1,
|
||||
type="download_expiring",
|
||||
title=long_title,
|
||||
data={"kind": "download_expiring", "title": long_title},
|
||||
)
|
||||
assert notif2.data["title"] == long_title
|
||||
|
||||
|
||||
def test_short_title_is_untouched():
|
||||
notif = create_notification(_CaptureDB(), user_id=1, type="t", title="short")
|
||||
assert notif.title == "short"
|
||||
|
||||
|
||||
def test_title_at_exactly_the_limit_is_untouched():
|
||||
exact = "y" * TITLE_MAX_LEN
|
||||
notif = create_notification(_CaptureDB(), user_id=1, type="t", title=exact)
|
||||
assert notif.title == exact
|
||||
|
||||
|
||||
def test_limit_tracks_the_actual_column_width():
|
||||
# The clamp is derived from the model, so it can't silently drift from the column a
|
||||
# migration would change (which is exactly what re-introduces the truncation bug).
|
||||
assert TITLE_MAX_LEN == Notification.__table__.c.title.type.length
|
||||
|
||||
|
||||
def test_over_long_title_actually_commits_on_postgres(db):
|
||||
# The stub tests above prove the clamp *logic*; this proves the real fix end-to-end —
|
||||
# a caption-length title now COMMITS into varchar(TITLE_MAX_LEN) instead of raising
|
||||
# StringDataRightTruncation. Uses the opt-in Postgres lane (skipped with no DB).
|
||||
user = User(email="reel@example.com")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
notif = create_notification(
|
||||
db, user_id=user.id, type="download_expiring", title="z" * 2000
|
||||
)
|
||||
db.refresh(notif)
|
||||
assert len(notif.title) == TITLE_MAX_LEN # persisted, not truncated by the DB
|
||||
@@ -0,0 +1,332 @@
|
||||
"""Regression (E5 S5b): the info-page rich detail (IMDb score/id, tagline, full cast) is persisted at
|
||||
rich-fetch time and served from the DB row instead of a live plex.metadata() call.
|
||||
|
||||
- `rich_detail`/`apply_rich_detail` are pure — parse + persist the rich-only fields.
|
||||
- `_enrich_people_batch` (the cast backfill) now also stamps those fields.
|
||||
- `item_detail` serves a MOVIE's detail from the row (survives a live-Plex outage) while keeping
|
||||
markers/streams live; an EPISODE (never enriched) still parses detail from the live meta.
|
||||
- `show_detail` serves from the row; `related` stays live best-effort.
|
||||
|
||||
The endpoint tests use the DB lane (`db` fixture) — skipped when no Postgres is reachable.
|
||||
"""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.routes.plex.browse as browse
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import PlexItem, PlexLibrary, PlexShow, User
|
||||
from app.plex import detail
|
||||
from app.plex.sync import _apply_facet_fields, _enrich_people_batch
|
||||
|
||||
# A full Plex metadata dict: an explicit IMDb Rating entry (must win over audienceRating), an IMDb Guid,
|
||||
# a tagline, and a two-person cast (one with a CDN headshot → proxied, one without → thumb None).
|
||||
RICH_META = {
|
||||
"Rating": [
|
||||
{"image": "imdb://image.rating.imdb", "value": 8.7},
|
||||
{"image": "themoviedb://image.rating", "value": 7.0},
|
||||
],
|
||||
"audienceRating": 9.9, # ignored: the imdb Rating entry is preferred
|
||||
"Guid": [{"id": "tmdb://238"}, {"id": "imdb://tt0068646"}],
|
||||
"contentRating": "R",
|
||||
"Genre": [{"tag": "Crime"}, {"tag": "Drama"}],
|
||||
"Director": [{"tag": "Francis Ford Coppola"}],
|
||||
"studio": "Paramount",
|
||||
"tagline": "An offer you can't refuse.",
|
||||
"Role": [
|
||||
{"tag": "Marlon Brando", "role": "Vito Corleone", "thumb": "https://metadata-static.plex.tv/b.jpg"},
|
||||
{"tag": "Al Pacino", "role": "Michael"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ---- pure parse/persist --------------------------------------------------------------------------
|
||||
|
||||
def test_rich_detail_prefers_imdb_entry_and_builds_link():
|
||||
d = detail.rich_detail(RICH_META)
|
||||
assert d["imdb_rating"] == 8.7 # the imdb Rating entry, not audienceRating (9.9)
|
||||
assert d["imdb_id"] == "tt0068646"
|
||||
assert d["imdb_url"] == "https://www.imdb.com/title/tt0068646/"
|
||||
assert d["tagline"] == "An offer you can't refuse."
|
||||
assert [c["name"] for c in d["cast"]] == ["Marlon Brando", "Al Pacino"]
|
||||
assert d["cast"][0]["role"] == "Vito Corleone"
|
||||
assert d["cast"][0]["thumb"] and d["cast"][0]["thumb"].startswith("/api/plex/person-image")
|
||||
assert d["cast"][1]["thumb"] is None # no CDN headshot → not proxied
|
||||
|
||||
|
||||
def test_rich_detail_is_imdb_only_no_audience_fallback():
|
||||
# E5 S6: imdb_rating is IMDb-ONLY. With no imdb:// Rating entry it's NULL — the Plex audience score
|
||||
# is NOT mislabelled as an IMDb score (it's served separately via the `rating` column).
|
||||
d = detail.rich_detail({"audienceRating": 7.25})
|
||||
assert d["imdb_rating"] is None
|
||||
assert d["imdb_id"] is None and d["imdb_url"] is None
|
||||
|
||||
|
||||
def test_apply_rich_detail_writes_only_rich_fields():
|
||||
row = SimpleNamespace(imdb_rating=None, imdb_id=None, tagline=None, full_cast=None)
|
||||
detail.apply_rich_detail(row, RICH_META)
|
||||
assert row.imdb_rating == 8.7
|
||||
assert row.imdb_id == "tt0068646"
|
||||
assert row.tagline == "An offer you can't refuse."
|
||||
assert [c["name"] for c in row.full_cast] == ["Marlon Brando", "Al Pacino"]
|
||||
|
||||
|
||||
def test_apply_rich_detail_empty_cast_persists_null():
|
||||
row = SimpleNamespace(imdb_rating=1.0, imdb_id="x", tagline="t", full_cast=[{"name": "stale"}])
|
||||
detail.apply_rich_detail(row, {}) # a poor/empty meta must clear to NULL, not keep stale data
|
||||
assert row.full_cast is None
|
||||
assert row.imdb_rating is None and row.imdb_id is None and row.tagline is None
|
||||
|
||||
|
||||
# ---- backfill persistence (DB lane) --------------------------------------------------------------
|
||||
|
||||
class _StubClient:
|
||||
"""Stand-in for PlexClient: `_enrich_people_batch` calls plex.metadata(rating_key); item_detail
|
||||
uses it as a context manager and passes markers=True. `raise_` simulates a Plex outage."""
|
||||
|
||||
def __init__(self, meta: dict | None = None, raise_: bool = False):
|
||||
self._meta = meta or {}
|
||||
self._raise = raise_
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def metadata(self, rating_key, markers=False):
|
||||
if self._raise:
|
||||
raise RuntimeError("plex unavailable")
|
||||
return self._meta
|
||||
|
||||
|
||||
def test_apply_facet_fields_preserves_enriched_cast_names():
|
||||
"""A full sync must NOT downgrade an already-enriched full cast back to the listing top-20 — since
|
||||
sync never clears people_enriched_at, nothing would regrow it and the person-page filter would lose
|
||||
the names past the listing cap (regression guard for E5 S5b's enrich gating)."""
|
||||
# The cheap listing's top few cast + the (always-present) director.
|
||||
thin = {"Role": [{"tag": "Al Pacino"}, {"tag": "James Caan"}], "Director": [{"tag": "Francis Ford Coppola"}]}
|
||||
|
||||
# Not-yet-enriched row: the listing cast IS applied.
|
||||
fresh = SimpleNamespace(people_enriched_at=None, directors=None, cast_names=None)
|
||||
_apply_facet_fields(fresh, thin)
|
||||
assert fresh.cast_names == ["Al Pacino", "James Caan"]
|
||||
|
||||
# Enriched row: the fuller stored cast is KEPT, not overwritten by the thin listing.
|
||||
from datetime import datetime, timezone
|
||||
enriched = SimpleNamespace(
|
||||
people_enriched_at=datetime.now(timezone.utc),
|
||||
directors=["Francis Ford Coppola"],
|
||||
cast_names=["Marlon Brando", "Al Pacino", "James Caan", "Diane Keaton"],
|
||||
)
|
||||
_apply_facet_fields(enriched, thin)
|
||||
assert enriched.cast_names == ["Marlon Brando", "Al Pacino", "James Caan", "Diane Keaton"]
|
||||
# people_text is rebuilt from the (updated) directors + the PRESERVED full cast.
|
||||
assert "Diane Keaton" in enriched.people_text and "Francis Ford Coppola" in enriched.people_text
|
||||
|
||||
|
||||
def test_apply_facet_fields_keeps_over_cap_cast_after_a_watermark_reset():
|
||||
"""The enriched-cast guard must survive a watermark RESET (people_enriched_at nulled, e.g. by a
|
||||
migration like 0065). A stored cast LONGER than the listing cap can only be an enriched fuller list,
|
||||
so it's kept even though the row now reads as un-enriched — comparing against the cap, not the
|
||||
current listing's length, is what makes this robust (regression guard for the review fix)."""
|
||||
thin = {"Role": [{"tag": f"Actor {i}"} for i in range(20)]} # a full 20-name listing
|
||||
over_cap = SimpleNamespace(
|
||||
people_enriched_at=None, # watermark was reset
|
||||
directors=None,
|
||||
cast_names=[f"Enriched {i}" for i in range(22)], # 22 > the 20 cap => enriched
|
||||
)
|
||||
_apply_facet_fields(over_cap, thin)
|
||||
assert over_cap.cast_names == [f"Enriched {i}" for i in range(22)] # kept, not downgraded
|
||||
|
||||
|
||||
def test_apply_facet_fields_tracks_a_trimmed_listing_when_unenriched():
|
||||
"""A listing-sourced cast (at or below the cap, never enriched) must TRACK Plex cast edits in both
|
||||
directions. When Plex trims the cast, the shorter fresh listing replaces the stored one — the old
|
||||
length-vs-this-listing guard would have wrongly kept the stale longer cast."""
|
||||
stored = SimpleNamespace(
|
||||
people_enriched_at=None,
|
||||
directors=None,
|
||||
cast_names=["A", "B", "C", "D", "E"], # 5, at/below the cap => listing-sourced
|
||||
)
|
||||
trimmed = {"Role": [{"tag": "A"}, {"tag": "B"}]} # Plex removed 3 cast members
|
||||
_apply_facet_fields(stored, trimmed)
|
||||
assert stored.cast_names == ["A", "B"]
|
||||
|
||||
|
||||
def test_enrich_people_batch_persists_detail(db):
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
mv = PlexItem(rating_key="m1", library_id=lib.id, kind="movie", title="The Godfather")
|
||||
db.add(mv)
|
||||
db.commit()
|
||||
assert mv.people_enriched_at is None and mv.full_cast is None
|
||||
|
||||
n = _enrich_people_batch(db, _StubClient(RICH_META))
|
||||
assert n == 1
|
||||
db.refresh(mv)
|
||||
assert mv.imdb_rating == 8.7
|
||||
assert mv.imdb_id == "tt0068646"
|
||||
assert mv.tagline == "An offer you can't refuse."
|
||||
assert [c["name"] for c in mv.full_cast] == ["Marlon Brando", "Al Pacino"]
|
||||
assert mv.cast_names == ["Marlon Brando", "Al Pacino"] # people enrich still runs alongside
|
||||
assert mv.people_enriched_at is not None # stamped so the backfill skips it next run
|
||||
|
||||
|
||||
# ---- item_detail serving (DB lane) ---------------------------------------------------------------
|
||||
|
||||
def _get(db, user, monkeypatch, path):
|
||||
monkeypatch.setattr(state, "_configured_cache", True) # pass the request gate
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
return TestClient(app).get(path)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def _user(db, email):
|
||||
u = User(email=email)
|
||||
db.add(u)
|
||||
db.flush()
|
||||
return u
|
||||
|
||||
|
||||
def test_item_detail_movie_serves_from_row_despite_live_outage(db, monkeypatch):
|
||||
"""An enriched movie serves cast/IMDb/tagline from the DB row even when the live Plex call raises —
|
||||
only markers/streams (which need the live meta) degrade."""
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
from datetime import datetime, timezone
|
||||
mv = PlexItem(
|
||||
rating_key="m1", library_id=lib.id, kind="movie", title="The Godfather",
|
||||
genres=["Crime", "Drama"], directors=["Francis Ford Coppola"], studio="Paramount",
|
||||
content_rating="R", imdb_rating=8.7, imdb_id="tt0068646",
|
||||
tagline="An offer you can't refuse.",
|
||||
full_cast=[{"name": "Marlon Brando", "role": "Vito Corleone", "thumb": None}],
|
||||
people_enriched_at=datetime.now(timezone.utc),
|
||||
)
|
||||
db.add(mv)
|
||||
db.commit()
|
||||
user = _user(db, "outage@example.com")
|
||||
db.commit()
|
||||
|
||||
monkeypatch.setattr(browse, "PlexClient", lambda db: _StubClient(raise_=True))
|
||||
resp = _get(db, user, monkeypatch, "/api/plex/item/m1")
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
assert body["imdb_rating"] == 8.7
|
||||
assert body["imdb_id"] == "tt0068646"
|
||||
assert body["imdb_url"] == "https://www.imdb.com/title/tt0068646/"
|
||||
assert body["tagline"] == "An offer you can't refuse."
|
||||
assert [c["name"] for c in body["cast"]] == ["Marlon Brando"]
|
||||
assert body["genres"] == ["Crime", "Drama"]
|
||||
assert body["markers"] == [] # live fetch failed → markers degrade, but detail survived
|
||||
|
||||
|
||||
def test_item_detail_movie_lazy_fills_when_unenriched(db, monkeypatch):
|
||||
"""A not-yet-enriched movie is filled from the live meta on view (and persisted for next time)."""
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
mv = PlexItem(rating_key="m1", library_id=lib.id, kind="movie", title="The Godfather")
|
||||
db.add(mv)
|
||||
db.commit()
|
||||
user = _user(db, "lazy@example.com")
|
||||
db.commit()
|
||||
|
||||
monkeypatch.setattr(browse, "PlexClient", lambda db: _StubClient(RICH_META))
|
||||
resp = _get(db, user, monkeypatch, "/api/plex/item/m1")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["imdb_id"] == "tt0068646"
|
||||
db.refresh(mv)
|
||||
assert mv.imdb_id == "tt0068646" # persisted
|
||||
assert mv.people_enriched_at is not None
|
||||
assert [c["name"] for c in mv.full_cast] == ["Marlon Brando", "Al Pacino"]
|
||||
|
||||
|
||||
def test_item_detail_episode_serves_rich_from_live(db, monkeypatch):
|
||||
"""Episodes are never enriched (not in the grid, no HU overlay), so their detail is parsed from the
|
||||
live meta the endpoint fetches for markers/streams."""
|
||||
lib = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
show = PlexShow(rating_key="s1", library_id=lib.id, title="A Show")
|
||||
db.add(show)
|
||||
db.flush()
|
||||
ep = PlexItem(
|
||||
rating_key="e1", library_id=lib.id, kind="episode", title="Pilot", show_id=show.id
|
||||
)
|
||||
db.add(ep)
|
||||
db.commit()
|
||||
user = _user(db, "ep@example.com")
|
||||
db.commit()
|
||||
|
||||
monkeypatch.setattr(browse, "PlexClient", lambda db: _StubClient(RICH_META))
|
||||
resp = _get(db, user, monkeypatch, "/api/plex/item/e1")
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
assert body["imdb_id"] == "tt0068646" # from the live meta, not the (NULL) row
|
||||
assert [c["name"] for c in body["cast"]] == ["Marlon Brando", "Al Pacino"]
|
||||
db.refresh(ep)
|
||||
assert ep.full_cast is None # episodes are not persisted/enriched
|
||||
|
||||
|
||||
def test_show_detail_serves_from_row(db, monkeypatch):
|
||||
"""A show serves cast/IMDb from the DB row; `related` failing (Plex down) must not drop the detail."""
|
||||
lib = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
from datetime import datetime, timezone
|
||||
show = PlexShow(
|
||||
rating_key="s1", library_id=lib.id, title="A Show", genres=["Drama"],
|
||||
imdb_rating=9.1, imdb_id="tt0903747",
|
||||
full_cast=[{"name": "Bryan Cranston", "role": "Walter White", "thumb": None}],
|
||||
people_enriched_at=datetime.now(timezone.utc),
|
||||
)
|
||||
db.add(show)
|
||||
db.commit()
|
||||
user = _user(db, "show@example.com")
|
||||
db.commit()
|
||||
|
||||
# _show_enrichment raises → related degrades to []; the row-served detail must still render.
|
||||
def _boom(db_, rk):
|
||||
raise RuntimeError("plex unavailable")
|
||||
|
||||
monkeypatch.setattr(browse, "_show_enrichment", _boom)
|
||||
resp = _get(db, user, monkeypatch, "/api/plex/show/s1")
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()["show"]
|
||||
assert body["imdb_rating"] == 9.1
|
||||
assert body["imdb_id"] == "tt0903747"
|
||||
assert body["imdb_url"] == "https://www.imdb.com/title/tt0903747/"
|
||||
assert [c["name"] for c in body["cast"]] == ["Bryan Cranston"]
|
||||
assert resp.json()["related"] == []
|
||||
|
||||
|
||||
def test_show_detail_lazy_fills_when_unenriched(db, monkeypatch):
|
||||
"""A not-yet-enriched show is filled from the live meta on first view (imdb/full_cast persisted +
|
||||
stamped) so subsequent views serve from the row."""
|
||||
lib = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
show = PlexShow(rating_key="s1", library_id=lib.id, title="A Show")
|
||||
db.add(show)
|
||||
db.commit()
|
||||
user = _user(db, "showfill@example.com")
|
||||
db.commit()
|
||||
|
||||
# _show_enrichment returns (live meta, no related) → the fill branch runs.
|
||||
monkeypatch.setattr(browse, "_show_enrichment", lambda db_, rk: (RICH_META, []))
|
||||
resp = _get(db, user, monkeypatch, "/api/plex/show/s1")
|
||||
assert resp.status_code == 200, resp.text
|
||||
assert resp.json()["show"]["imdb_id"] == "tt0068646"
|
||||
db.refresh(show)
|
||||
assert show.imdb_id == "tt0068646" # persisted for next time
|
||||
assert show.people_enriched_at is not None
|
||||
assert [c["name"] for c in show.full_cast] == ["Marlon Brando", "Al Pacino"]
|
||||
@@ -0,0 +1,132 @@
|
||||
"""S2 · field-scoped search — the `field` param on /api/plex/library.
|
||||
|
||||
The search box's scope-pill narrows a full-text query to ONE facet of the row: Everywhere (`all`,
|
||||
the untouched weighted `search_vector`), Titles, Cast & crew (`people`), Descriptions (`summary`),
|
||||
or Episodes (episode rows only, title-only). These lock that each scope is a strict narrowing of
|
||||
Everywhere, that `all` is unchanged from the pre-scope behaviour, and that the Episode scope empties
|
||||
the movie/show grid and only applies where shows are in scope. DB lane (`db` fixture) — skipped
|
||||
without Postgres (the `search_vector` GIN + per-field `to_tsvector` recheck are real Postgres FTS).
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import (
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
PlexSeason,
|
||||
PlexShow,
|
||||
User,
|
||||
)
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
def _dt(day: int) -> datetime:
|
||||
return datetime(2024, 2, day, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _seed(db):
|
||||
"""One rare token, `zephyr`, planted in exactly one facet per row so a scope's narrowing is
|
||||
unambiguous: mt has it ONLY in the title, mp ONLY in cast (people_text), ms ONLY in the summary,
|
||||
and episode ep ONLY in its title (its show carries none). Everywhere matches all four; each
|
||||
scoped field matches exactly its one row."""
|
||||
movie_lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
show_lib = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||
db.add_all([movie_lib, show_lib])
|
||||
db.flush()
|
||||
mt = PlexItem(rating_key="mt", library_id=movie_lib.id, kind="movie",
|
||||
title="Zephyr Rising", people_text="Alice Stone", summary="A calm coastal day.",
|
||||
year=2001, added_at=_dt(1))
|
||||
mp = PlexItem(rating_key="mp", library_id=movie_lib.id, kind="movie",
|
||||
title="Quiet Harbor", people_text="Zephyr Vance", summary="A calm coastal day.",
|
||||
year=2002, added_at=_dt(2))
|
||||
ms = PlexItem(rating_key="ms", library_id=movie_lib.id, kind="movie",
|
||||
title="Quiet Harbor Two", people_text="Alice Stone",
|
||||
summary="A story about zephyr winds.", year=2003, added_at=_dt(3))
|
||||
show = PlexShow(rating_key="s1", library_id=show_lib.id, title="Nimbus Show",
|
||||
summary="Weather anthology.", people_text="Bob Cloud",
|
||||
year=2004, added_at=_dt(4), child_count=1)
|
||||
db.add_all([mt, mp, ms, show])
|
||||
db.flush()
|
||||
season = PlexSeason(rating_key="se1", show_id=show.id, season_number=1)
|
||||
db.add(season)
|
||||
db.flush()
|
||||
ep = PlexItem(rating_key="ep", library_id=show_lib.id, kind="episode", title="Zephyr Episode",
|
||||
show_id=show.id, season_id=season.id, season_number=1, episode_number=1,
|
||||
added_at=_dt(5))
|
||||
db.add(ep)
|
||||
user = User(email="plexfield@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return {"user": user}
|
||||
|
||||
|
||||
def _get(db, user, monkeypatch, path):
|
||||
monkeypatch.setattr(state, "_configured_cache", True)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
return TestClient(app).get(path)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def _ids(body):
|
||||
return sorted(i["id"] for i in body["items"])
|
||||
|
||||
|
||||
class TestFieldScopedSearch:
|
||||
def test_everywhere_matches_every_facet(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?q=zephyr&field=all").json()
|
||||
# title (mt), cast (mp), summary (ms) all land in the item grid; the episode is grouped out
|
||||
assert _ids(body) == ["mp", "ms", "mt"]
|
||||
assert [e["title"] for e in body["episodes"]] == ["Zephyr Episode"]
|
||||
|
||||
def test_field_all_is_identical_to_default(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
default = _get(db, s["user"], monkeypatch, "/api/plex/library?q=zephyr").json()
|
||||
scoped = _get(db, s["user"], monkeypatch, "/api/plex/library?q=zephyr&field=all").json()
|
||||
assert _ids(default) == _ids(scoped) == ["mp", "ms", "mt"]
|
||||
assert default["total"] == scoped["total"]
|
||||
assert [e["id"] for e in default["episodes"]] == [e["id"] for e in scoped["episodes"]]
|
||||
|
||||
def test_title_scope_narrows_to_title_match(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?q=zephyr&field=title").json()
|
||||
assert _ids(body) == ["mt"]
|
||||
assert body["total"] == 1
|
||||
assert body["episodes"] == [] # non-episode field scopes suppress the episode section
|
||||
|
||||
def test_people_scope_narrows_to_cast_match(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?q=zephyr&field=people").json()
|
||||
assert _ids(body) == ["mp"]
|
||||
|
||||
def test_summary_scope_narrows_to_description_match(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?q=zephyr&field=summary").json()
|
||||
assert _ids(body) == ["ms"]
|
||||
|
||||
def test_episode_scope_empties_grid_and_returns_episode(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?q=zephyr&field=episode").json()
|
||||
assert body["items"] == []
|
||||
assert body["total"] == 0
|
||||
assert [e["title"] for e in body["episodes"]] == ["Zephyr Episode"]
|
||||
|
||||
def test_episode_scope_needs_shows_in_scope(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(
|
||||
db, s["user"], monkeypatch, "/api/plex/library?q=zephyr&field=episode&scope=movie"
|
||||
).json()
|
||||
# movie-only scope has no show libraries -> no episodes to surface, and the grid stays empty
|
||||
assert body["items"] == []
|
||||
assert body["episodes"] == []
|
||||
|
||||
def test_unknown_field_falls_back_to_everywhere(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?q=zephyr&field=bogus").json()
|
||||
assert _ids(body) == ["mp", "ms", "mt"]
|
||||
@@ -0,0 +1,125 @@
|
||||
"""E5 S6: HU-overlay language serving. When ?lang=hu, the unified library, facets, genre filter and
|
||||
detail endpoints serve the Hungarian `*_hu` columns (falling back to EN where a row has no HU twin);
|
||||
?lang=en (the default) serves English. Also covers the imdb_rating IMDb-only / audience-score split.
|
||||
DB lane (`db` fixture) — skipped without Postgres.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import PlexItem, PlexLibrary, User
|
||||
|
||||
|
||||
def _seed(db):
|
||||
"""One EN movie library with a single movie that has both EN and HU metadata, plus a second movie
|
||||
that is EN-only (no HU twin) to prove the coalesce fallback."""
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
m1 = PlexItem(
|
||||
rating_key="m1", library_id=lib.id, kind="movie",
|
||||
title="English Title", title_hu="Magyar Cím",
|
||||
summary="English summary.", summary_hu="Magyar összefoglaló.",
|
||||
genres=["Drama"], genres_hu=["Dráma"],
|
||||
rating=6.5, imdb_rating=None,
|
||||
year=2001, added_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
people_enriched_at=datetime(2024, 1, 1, tzinfo=timezone.utc), # serve from row, skip live enrich
|
||||
)
|
||||
m2 = PlexItem(
|
||||
rating_key="m2", library_id=lib.id, kind="movie",
|
||||
title="EN Only", genres=["Comedy"], # no HU twin
|
||||
year=2002, added_at=datetime(2024, 1, 2, tzinfo=timezone.utc),
|
||||
people_enriched_at=datetime(2024, 1, 2, tzinfo=timezone.utc),
|
||||
)
|
||||
db.add_all([m1, m2])
|
||||
user = User(email="plexlang@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
return user
|
||||
|
||||
|
||||
def _get(db, user, monkeypatch, path):
|
||||
monkeypatch.setattr(state, "_configured_cache", True) # the request gate 503s an unconfigured app
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
client = TestClient(app) # no context manager: skip lifespan
|
||||
return client.get(path)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def _titles(resp):
|
||||
return {i["title"] for i in resp.json()["items"]}
|
||||
|
||||
|
||||
def test_library_serves_active_language_titles(db, monkeypatch):
|
||||
user = _seed(db)
|
||||
hu = _get(db, user, monkeypatch, "/api/plex/library?scope=movie&lang=hu")
|
||||
en = _get(db, user, monkeypatch, "/api/plex/library?scope=movie&lang=en")
|
||||
# HU serves the twin's Hungarian title; the EN-only row falls back to its English title.
|
||||
assert _titles(hu) == {"Magyar Cím", "EN Only"}
|
||||
assert _titles(en) == {"English Title", "EN Only"}
|
||||
|
||||
|
||||
def test_facets_vocabulary_is_language_specific(db, monkeypatch):
|
||||
user = _seed(db)
|
||||
hu = _get(db, user, monkeypatch, "/api/plex/facets?scope=movie&lang=hu")
|
||||
en = _get(db, user, monkeypatch, "/api/plex/facets?scope=movie&lang=en")
|
||||
hu_genres = {g["value"] for g in hu.json()["genres"]}
|
||||
en_genres = {g["value"] for g in en.json()["genres"]}
|
||||
assert "Dráma" in hu_genres and "Drama" not in hu_genres
|
||||
assert "Drama" in en_genres and "Dráma" not in en_genres
|
||||
|
||||
|
||||
def test_genre_filter_matches_active_language_vocabulary(db, monkeypatch):
|
||||
user = _seed(db)
|
||||
# The HU genre chip filters the HU column; the same string against the EN column matches nothing.
|
||||
hit = _get(db, user, monkeypatch, "/api/plex/library?scope=movie&lang=hu&genres=Dráma")
|
||||
miss = _get(db, user, monkeypatch, "/api/plex/library?scope=movie&lang=en&genres=Dráma")
|
||||
assert _titles(hit) == {"Magyar Cím"}
|
||||
assert miss.json()["items"] == []
|
||||
|
||||
|
||||
def test_item_detail_language_and_rating_split(db, monkeypatch):
|
||||
user = _seed(db)
|
||||
resp = _get(db, user, monkeypatch, "/api/plex/item/m1?lang=hu")
|
||||
body = resp.json()
|
||||
assert body["title"] == "Magyar Cím"
|
||||
assert body["summary"] == "Magyar összefoglaló."
|
||||
# imdb_rating is IMDb-only (None here); the audience score is served separately as `rating`.
|
||||
assert body["imdb_rating"] is None
|
||||
assert body["rating"] == 6.5
|
||||
|
||||
|
||||
def test_hu_search_still_finds_en_only_titles(db, monkeypatch):
|
||||
user = _seed(db)
|
||||
# m2 ("EN Only") has no HU twin, so search_vector_hu carries no title tokens for it. HU-mode search
|
||||
# must still match it via the EN vector (vector_match's union) — findable by its English title.
|
||||
resp = _get(db, user, monkeypatch, "/api/plex/library?scope=movie&lang=hu&q=Only")
|
||||
assert "EN Only" in _titles(resp)
|
||||
|
||||
|
||||
def test_rating_filter_prefers_imdb_over_audience(db, monkeypatch):
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
# imdb 8.0 but audience 4.0 → a rating_min of 7 must MATCH (filter the displayed best score,
|
||||
# IMDb preferred), where filtering the raw audience column (4.0) would wrongly exclude it.
|
||||
db.add(PlexItem(
|
||||
rating_key="r1", library_id=lib.id, kind="movie", title="High IMDb",
|
||||
imdb_rating=8.0, rating=4.0, year=2001,
|
||||
added_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
people_enriched_at=datetime(2024, 1, 1, tzinfo=timezone.utc),
|
||||
))
|
||||
user = User(email="rating@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
hit = _get(db, user, monkeypatch, "/api/plex/library?scope=movie&rating_min=7")
|
||||
assert "High IMDb" in _titles(hit)
|
||||
miss = _get(db, user, monkeypatch, "/api/plex/library?scope=movie&rating_min=9")
|
||||
assert "High IMDb" not in _titles(miss)
|
||||
@@ -0,0 +1,223 @@
|
||||
"""DB-backed regression (E5 S5a): the HU-overlay sync writes Hungarian metadata onto the matched EN
|
||||
rows (paired by guid), never creating browsable HU rows, and overlay libraries are excluded from the
|
||||
browse scope. Also covers the canonical-guid parse and that the generated search_vector_hu populates.
|
||||
|
||||
Uses the DB lane (`db` fixture) — skipped when no Postgres is reachable. Runs against the real schema
|
||||
(migrations 0063/0064) so the JSONB + STORED generated columns are exercised.
|
||||
"""
|
||||
from sqlalchemy import func, select
|
||||
|
||||
import app.plex.sync as syncmod
|
||||
from app import sysconfig
|
||||
from app.models import PlexItem, PlexLibrary
|
||||
from app.plex.sync import _apply_item, _canonical_guid, _sync_overlay
|
||||
from app.routes.plex.browse import _content_libs
|
||||
|
||||
|
||||
class _StubPlex:
|
||||
"""Minimal PlexClient stand-in: _sync_overlay only calls section_items via _paginate."""
|
||||
|
||||
def __init__(self, items: list[dict]):
|
||||
self._items = items
|
||||
|
||||
def section_items(self, key, item_type, start, size):
|
||||
return self._items[start : start + size], len(self._items)
|
||||
|
||||
|
||||
def _hu_movie(rk: str, title: str, summary: str, genres: list[str], guid: str | None) -> dict:
|
||||
meta = {"ratingKey": rk, "title": title, "summary": summary, "Genre": [{"tag": g} for g in genres]}
|
||||
if guid:
|
||||
meta["Guid"] = [{"id": guid}]
|
||||
return meta
|
||||
|
||||
|
||||
def test_canonical_guid_prefers_tmdb_then_imdb():
|
||||
assert _canonical_guid({"Guid": [{"id": "imdb://tt1"}, {"id": "tmdb://55"}]}) == "tmdb:55"
|
||||
assert _canonical_guid({"Guid": [{"id": "imdb://tt9"}, {"id": "tvdb://7"}]}) == "imdb:tt9"
|
||||
assert _canonical_guid({"Guid": []}) is None
|
||||
assert _canonical_guid({}) is None
|
||||
|
||||
|
||||
def test_sync_overlay_writes_hu_onto_matched_en_rows(db):
|
||||
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||
hu_lib = PlexLibrary(plex_key="9", title="Filmek", kind="movie", metadata_lang="hu")
|
||||
db.add_all([en_lib, hu_lib])
|
||||
db.flush()
|
||||
hu_lib.overlay_of = en_lib.id
|
||||
matched = PlexItem(
|
||||
rating_key="m1", library_id=en_lib.id, kind="movie", title="Heat", guid="tmdb:1"
|
||||
)
|
||||
unmatched = PlexItem(
|
||||
rating_key="m2", library_id=en_lib.id, kind="movie", title="Solo", guid="tmdb:2"
|
||||
)
|
||||
db.add_all([matched, unmatched])
|
||||
db.commit()
|
||||
|
||||
stub = _StubPlex(
|
||||
[
|
||||
_hu_movie("h1", "Öldöklés", "Egy rablás terve.", ["Akció", "Krimi"], "tmdb://1"),
|
||||
_hu_movie("h2", "Ismeretlen", "Nincs EN párja.", ["Dráma"], "tmdb://999"),
|
||||
]
|
||||
)
|
||||
stats = {"overlaid": 0}
|
||||
_sync_overlay(db, stub, hu_lib, en_lib.id, stats)
|
||||
db.commit()
|
||||
|
||||
db.refresh(matched)
|
||||
db.refresh(unmatched)
|
||||
assert matched.title_hu == "Öldöklés"
|
||||
assert matched.summary_hu == "Egy rablás terve."
|
||||
assert matched.genres_hu == ["Akció", "Krimi"]
|
||||
assert matched.search_vector_hu is not None # the STORED generated column populated from title_hu
|
||||
# The unmatched EN row keeps NULL HU fields (serving falls back to EN).
|
||||
assert unmatched.title_hu is None
|
||||
assert stats["overlaid"] == 1
|
||||
# No browsable rows were created — the HU items never became plex_items of the overlay library.
|
||||
hu_row_count = db.scalar(
|
||||
select(func.count(PlexItem.id)).where(PlexItem.library_id == hu_lib.id)
|
||||
)
|
||||
assert hu_row_count == 0
|
||||
# The EN library still holds exactly its two movies (no dup from the overlay pass).
|
||||
en_row_count = db.scalar(
|
||||
select(func.count(PlexItem.id)).where(PlexItem.library_id == en_lib.id)
|
||||
)
|
||||
assert en_row_count == 2
|
||||
|
||||
|
||||
class _StubClient:
|
||||
"""Full-sync stand-in: one EN movie section (key 4) + its HU twin (key 9), paired by guid."""
|
||||
|
||||
def __init__(self, db=None):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def sections(self):
|
||||
return [
|
||||
{"key": "4", "title": "Movies", "type": "movie"},
|
||||
{"key": "9", "title": "Filmek", "type": "movie"},
|
||||
]
|
||||
|
||||
def section_items(self, key, item_type, start, size):
|
||||
if item_type != syncmod._TYPE_MOVIE:
|
||||
return [], 0
|
||||
if key == "4":
|
||||
items = [{"ratingKey": "en1", "title": "Heat", "Guid": [{"id": "tmdb://1"}]}]
|
||||
elif key == "9":
|
||||
items = [
|
||||
{
|
||||
"ratingKey": "hu1", "title": "Öldöklés", "summary": "Rablás.",
|
||||
"Genre": [{"tag": "Akció"}], "Guid": [{"id": "tmdb://1"}],
|
||||
}
|
||||
]
|
||||
else:
|
||||
items = []
|
||||
return items[start : start + size], len(items)
|
||||
|
||||
def collections(self, key):
|
||||
return []
|
||||
|
||||
def metadata(self, rk, markers=False):
|
||||
return {}
|
||||
|
||||
|
||||
def test_sync_designates_overlay_and_writes_hu(db, monkeypatch):
|
||||
sysconfig.set_value(db, "plex_enabled", True)
|
||||
sysconfig.set_value(db, "plex_libraries", "4") # only the EN section is browsable
|
||||
sysconfig.set_value(db, "plex_overlay_map", "9:4") # HU section 9 overlays EN section 4
|
||||
db.commit()
|
||||
monkeypatch.setattr(syncmod, "PlexClient", _StubClient)
|
||||
|
||||
stats = syncmod.sync(db)
|
||||
|
||||
en = db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == "4")).scalars().one()
|
||||
hu = db.execute(select(PlexLibrary).where(PlexLibrary.plex_key == "9")).scalars().one()
|
||||
assert en.metadata_lang == "en" and en.overlay_of is None
|
||||
assert hu.metadata_lang == "hu" and hu.overlay_of == en.id # pass-2 designation
|
||||
# The EN movie captured its guid (from includeGuids listing) and received the HU overlay.
|
||||
m = db.execute(select(PlexItem).where(PlexItem.rating_key == "en1")).scalars().one()
|
||||
assert m.guid == "tmdb:1"
|
||||
assert m.title_hu == "Öldöklés"
|
||||
assert m.genres_hu == ["Akció"]
|
||||
# The HU section is not browsable and mirrored no rows of its own.
|
||||
movie_ids, _ = _content_libs(db)
|
||||
assert en.id in movie_ids and hu.id not in movie_ids
|
||||
assert db.scalar(select(func.count(PlexItem.id)).where(PlexItem.library_id == hu.id)) == 0
|
||||
assert stats["overlaid"] == 1
|
||||
|
||||
|
||||
def test_overlay_reconcile_clears_stale_hu_when_twin_vanishes(db):
|
||||
"""A re-run whose HU section no longer carries a title's guid must drop that EN row's stale HU."""
|
||||
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||
hu_lib = PlexLibrary(plex_key="9", title="Filmek", kind="movie", metadata_lang="hu")
|
||||
db.add_all([en_lib, hu_lib])
|
||||
db.flush()
|
||||
hu_lib.overlay_of = en_lib.id
|
||||
row = PlexItem(
|
||||
rating_key="m1", library_id=en_lib.id, kind="movie", title="Heat",
|
||||
guid="tmdb:1", title_hu="Öldöklés", summary_hu="régi", genres_hu=["Akció"],
|
||||
)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
# The HU section no longer lists this film (empty overlay).
|
||||
_sync_overlay(db, _StubPlex([]), hu_lib, en_lib.id, {"overlaid": 0})
|
||||
db.commit()
|
||||
db.refresh(row)
|
||||
assert row.title_hu is None and row.summary_hu is None and row.genres_hu is None
|
||||
|
||||
|
||||
def test_apply_item_keeps_prior_guid_when_listing_lacks_guid():
|
||||
row = PlexItem(rating_key="m1", kind="movie", guid="tmdb:1")
|
||||
_apply_item(row, lib_id=7, kind="movie", meta={"title": "Heat"}) # no Guid array
|
||||
assert row.guid == "tmdb:1" # the prior guid survives a guid-less listing
|
||||
_apply_item(row, lib_id=7, kind="movie", meta={"title": "Heat", "Guid": [{"id": "tmdb://2"}]})
|
||||
assert row.guid == "tmdb:2" # a real guid in the listing still updates it
|
||||
|
||||
|
||||
def test_overlay_applies_to_all_editions_sharing_a_guid(db):
|
||||
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||
hu_lib = PlexLibrary(plex_key="9", title="Filmek", kind="movie", metadata_lang="hu")
|
||||
db.add_all([en_lib, hu_lib])
|
||||
db.flush()
|
||||
hu_lib.overlay_of = en_lib.id
|
||||
ed1 = PlexItem(rating_key="a", library_id=en_lib.id, kind="movie", title="Heat", guid="tmdb:1")
|
||||
ed2 = PlexItem(rating_key="b", library_id=en_lib.id, kind="movie", title="Heat (Director's Cut)", guid="tmdb:1")
|
||||
db.add_all([ed1, ed2])
|
||||
db.commit()
|
||||
|
||||
# Two HU entries share the guid — first wins, no double count.
|
||||
stub = _StubPlex(
|
||||
[
|
||||
_hu_movie("h1", "Öldöklés", "első", ["Akció"], "tmdb://1"),
|
||||
_hu_movie("h2", "Öldöklés (dup)", "második", ["Krimi"], "tmdb://1"),
|
||||
]
|
||||
)
|
||||
stats = {"overlaid": 0}
|
||||
_sync_overlay(db, stub, hu_lib, en_lib.id, stats)
|
||||
db.commit()
|
||||
db.refresh(ed1)
|
||||
db.refresh(ed2)
|
||||
assert ed1.title_hu == "Öldöklés" and ed2.title_hu == "Öldöklés" # both editions overlaid
|
||||
assert ed1.genres_hu == ["Akció"] and ed2.genres_hu == ["Akció"] # first HU entry won
|
||||
assert stats["overlaid"] == 1 # counted once for the guid, not per edition or per dup HU item
|
||||
|
||||
|
||||
def test_content_libs_excludes_overlay_libraries(db):
|
||||
en_lib = PlexLibrary(plex_key="4", title="Movies", kind="movie")
|
||||
en_shows = PlexLibrary(plex_key="1", title="TV Shows", kind="show")
|
||||
db.add_all([en_lib, en_shows])
|
||||
db.flush()
|
||||
overlay = PlexLibrary(
|
||||
plex_key="9", title="Filmek", kind="movie", metadata_lang="hu", overlay_of=en_lib.id
|
||||
)
|
||||
db.add(overlay)
|
||||
db.commit()
|
||||
|
||||
movie_ids, show_ids = _content_libs(db)
|
||||
assert en_lib.id in movie_ids
|
||||
assert overlay.id not in movie_ids # the HU overlay is never browsable
|
||||
assert show_ids == [en_shows.id]
|
||||
@@ -0,0 +1,114 @@
|
||||
"""DB-backed regression (E5 S4): GET /api/plex/person/{name} returns a cast/crew member's
|
||||
filmography — movies + shows in the ENABLED libraries whose cast OR directors include the name,
|
||||
each carrying the caller's watch-state, plus the cached headshot. Also covers `_content_libs`
|
||||
scoping (disabled libraries excluded) and the kind filter (episodes are not movies).
|
||||
|
||||
Uses the DB lane (`db` fixture) — skipped when no Postgres is reachable.
|
||||
"""
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import (
|
||||
PlexItem,
|
||||
PlexLibrary,
|
||||
PlexPerson,
|
||||
PlexShow,
|
||||
PlexState,
|
||||
User,
|
||||
)
|
||||
from app.routes.plex.browse import _content_libs
|
||||
|
||||
NAME = "Robert De Niro"
|
||||
|
||||
|
||||
def _get(db, user, monkeypatch, path):
|
||||
monkeypatch.setattr(state, "_configured_cache", True) # pass the request gate
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
return TestClient(app).get(path) # no lifespan (skip scheduler/HLS sweep)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def test_person_detail_returns_cast_and_director_credits(db, monkeypatch):
|
||||
movies_lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
shows_lib = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||
off_lib = PlexLibrary(plex_key="3", title="Off", kind="movie", enabled=False)
|
||||
db.add_all([movies_lib, shows_lib, off_lib])
|
||||
db.flush()
|
||||
|
||||
heat = PlexItem(
|
||||
rating_key="m1", library_id=movies_lib.id, kind="movie", title="Heat",
|
||||
year=1995, cast_names=[NAME, "Al Pacino"],
|
||||
)
|
||||
# Director-only credit: the name is NOT in cast_names, only directors → the OR must still catch it.
|
||||
directed = PlexItem(
|
||||
rating_key="m2", library_id=movies_lib.id, kind="movie", title="A Bronx Tale",
|
||||
year=1993, cast_names=["Chazz Palminteri"], directors=[NAME],
|
||||
)
|
||||
# Excluded: a movie in a DISABLED library.
|
||||
hidden = PlexItem(
|
||||
rating_key="m3", library_id=off_lib.id, kind="movie", title="Hidden",
|
||||
year=2000, cast_names=[NAME],
|
||||
)
|
||||
# Excluded: an episode (kind != "movie") that happens to list the name.
|
||||
episode = PlexItem(
|
||||
rating_key="e1", library_id=movies_lib.id, kind="episode", title="An Episode",
|
||||
year=2001, cast_names=[NAME],
|
||||
)
|
||||
show = PlexShow(rating_key="s1", library_id=shows_lib.id, title="A Show", cast_names=[NAME])
|
||||
db.add_all([heat, directed, hidden, episode, show])
|
||||
db.add(PlexPerson(name=NAME, photo_thumb="/img/deniro.jpg"))
|
||||
db.flush()
|
||||
|
||||
user = User(email="person@example.com")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
db.add(PlexState(user_id=user.id, item_id=heat.id, status="watched"))
|
||||
db.commit()
|
||||
|
||||
resp = _get(db, user, monkeypatch, f"/api/plex/person/{NAME.replace(' ', '%20')}")
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
|
||||
assert body["name"] == NAME
|
||||
assert body["photo"] == "/img/deniro.jpg"
|
||||
# Both the cast credit (Heat) and the director-only credit (A Bronx Tale), year desc; NOT the
|
||||
# disabled-library movie and NOT the episode.
|
||||
assert [m["title"] for m in body["movies"]] == ["Heat", "A Bronx Tale"]
|
||||
assert body["movies"][0]["status"] == "watched" # from the caller's PlexState
|
||||
assert body["movies"][1]["status"] == "new"
|
||||
assert all(m["type"] == "movie" for m in body["movies"])
|
||||
# The show credit.
|
||||
assert [s["title"] for s in body["shows"]] == ["A Show"]
|
||||
assert body["shows"][0]["type"] == "show"
|
||||
|
||||
|
||||
def test_person_detail_unknown_name_is_empty(db, monkeypatch):
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
db.add(PlexItem(rating_key="m1", library_id=lib.id, kind="movie", title="Solo", cast_names=["Nobody Else"]))
|
||||
user = User(email="empty@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
|
||||
resp = _get(db, user, monkeypatch, "/api/plex/person/Ghost%20Actor")
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
assert body == {"name": "Ghost Actor", "photo": None, "movies": [], "shows": []}
|
||||
|
||||
|
||||
def test_content_libs_partitions_only_enabled(db):
|
||||
db.add_all([
|
||||
PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True),
|
||||
PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True),
|
||||
PlexLibrary(plex_key="3", title="OffMovie", kind="movie", enabled=False),
|
||||
])
|
||||
db.commit()
|
||||
movie_ids, show_ids = _content_libs(db)
|
||||
assert len(movie_ids) == 1 and len(show_ids) == 1 # the disabled movie lib is excluded
|
||||
@@ -0,0 +1,357 @@
|
||||
"""S1 characterization net for the Plex-package db.query() -> SQLAlchemy 2.0 select() conversion.
|
||||
|
||||
These lock the OBSERVABLE behaviour of the previously-untested hot query paths — the unified_library
|
||||
union builder (+ its watch-state branches, ordering, search ranking, episode grouping), the rollup
|
||||
aggregate, and the sync upsert idempotency — so the mechanical 1.x -> 2.0 conversion can be proven
|
||||
behaviour-equivalent: the exact same assertions must pass before and after the conversion. DB lane
|
||||
(`db` fixture) — skipped without Postgres.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
import app.state as state
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.main import app
|
||||
from app.models import PlexItem, PlexLibrary, PlexSeason, PlexShow, PlexState, User
|
||||
from app.plex import rollup, sync
|
||||
|
||||
|
||||
def _dt(day: int) -> datetime:
|
||||
return datetime(2024, 1, day, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _seed(db):
|
||||
"""A minimal cross-library corpus: 2 movies, 1 show with 2 episodes, and a per-user watch-state
|
||||
where movie m1 is watched and episode e1 is in-progress (so the show rolls up to in_progress)."""
|
||||
movie_lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
show_lib = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||
db.add_all([movie_lib, show_lib])
|
||||
db.flush()
|
||||
m1 = PlexItem(rating_key="m1", library_id=movie_lib.id, kind="movie",
|
||||
title="Alpha Movie", year=2001, added_at=_dt(1))
|
||||
m2 = PlexItem(rating_key="m2", library_id=movie_lib.id, kind="movie",
|
||||
title="Bravo Movie", year=2002, added_at=_dt(2))
|
||||
show = PlexShow(rating_key="s1", library_id=show_lib.id, title="Charlie Show",
|
||||
year=2003, added_at=_dt(3), child_count=1)
|
||||
db.add_all([m1, m2, show])
|
||||
db.flush()
|
||||
season = PlexSeason(rating_key="se1", show_id=show.id, season_number=1)
|
||||
db.add(season)
|
||||
db.flush()
|
||||
e1 = PlexItem(rating_key="e1", library_id=show_lib.id, kind="episode", title="Charlie Pilot",
|
||||
show_id=show.id, season_id=season.id, season_number=1, episode_number=1, added_at=_dt(4))
|
||||
e2 = PlexItem(rating_key="e2", library_id=show_lib.id, kind="episode", title="Charlie Finale",
|
||||
show_id=show.id, season_id=season.id, season_number=1, episode_number=2, added_at=_dt(5))
|
||||
db.add_all([e1, e2])
|
||||
user = User(email="plexbrowse@example.com")
|
||||
db.add(user)
|
||||
db.flush()
|
||||
db.add_all([
|
||||
PlexState(user_id=user.id, item_id=m1.id, status="watched"),
|
||||
PlexState(user_id=user.id, item_id=e1.id, position_seconds=120),
|
||||
])
|
||||
db.commit()
|
||||
return {"user": user, "movie_lib": movie_lib, "show_lib": show_lib, "show": show}
|
||||
|
||||
|
||||
def _request(db, user, monkeypatch, method, path, json=None):
|
||||
monkeypatch.setattr(state, "_configured_cache", True) # the request gate 503s an unconfigured app
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
app.dependency_overrides[current_user] = lambda: user
|
||||
try:
|
||||
client = TestClient(app) # no context manager: skip lifespan (scheduler/HLS sweep)
|
||||
return client.request(method, path, json=json)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
|
||||
def _get(db, user, monkeypatch, path):
|
||||
return _request(db, user, monkeypatch, "GET", path)
|
||||
|
||||
|
||||
class TestUnifiedLibrary:
|
||||
def test_both_scope_unions_movies_and_shows_ordered_by_added(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
r = _get(db, s["user"], monkeypatch, "/api/plex/library?scope=both")
|
||||
assert r.status_code == 200, r.text
|
||||
body = r.json()
|
||||
assert body["total"] == 3 # 2 movies + 1 show, unioned
|
||||
# default sort = added_at desc: show (day 3) > bravo (day 2) > alpha (day 1)
|
||||
assert [i["id"] for i in body["items"]] == ["s1", "m2", "m1"]
|
||||
show_item = next(i for i in body["items"] if i["type"] == "show")
|
||||
assert show_item["status"] == "in_progress" # e1 in-progress -> show rollup
|
||||
assert show_item["season_count"] == 1
|
||||
assert next(i for i in body["items"] if i["id"] == "m1")["status"] == "watched"
|
||||
|
||||
def test_scope_movie_only(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?scope=movie").json()
|
||||
assert body["total"] == 2
|
||||
assert all(i["type"] == "movie" for i in body["items"])
|
||||
|
||||
def test_scope_show_only(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?scope=show").json()
|
||||
assert [i["id"] for i in body["items"]] == ["s1"]
|
||||
assert body["items"][0]["season_count"] == 1
|
||||
|
||||
def test_watch_state_branches(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
# watched: only movie m1 (the show is in_progress, not fully watched)
|
||||
watched = _get(db, s["user"], monkeypatch, "/api/plex/library?show=watched").json()
|
||||
assert [i["id"] for i in watched["items"]] == ["m1"]
|
||||
# in_progress: only the show (no movie is in progress)
|
||||
inprog = _get(db, s["user"], monkeypatch, "/api/plex/library?show=in_progress").json()
|
||||
assert [i["id"] for i in inprog["items"]] == ["s1"]
|
||||
# unwatched: only movie m2 (no state). The show is EXCLUDED — "unwatched" for a show means
|
||||
# 0 watched AND 0 in-progress episodes, and e1 is in-progress. (Locks the actual branch.)
|
||||
unwatched = _get(db, s["user"], monkeypatch, "/api/plex/library?show=unwatched").json()
|
||||
assert [i["id"] for i in unwatched["items"]] == ["m2"]
|
||||
|
||||
def test_search_matches_by_rank_and_groups_episodes(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?q=charlie").json()
|
||||
# only the show title matches in the item grid; the movies don't contain "charlie"
|
||||
assert [i["id"] for i in body["items"]] == ["s1"]
|
||||
assert body["total"] == 1
|
||||
# matching episodes come back grouped in the separate episodes list
|
||||
assert sorted(e["title"] for e in body["episodes"]) == ["Charlie Finale", "Charlie Pilot"]
|
||||
|
||||
def test_pagination_caps_items_not_total(self, db, monkeypatch):
|
||||
s = _seed(db)
|
||||
body = _get(db, s["user"], monkeypatch, "/api/plex/library?limit=2&offset=0").json()
|
||||
assert body["total"] == 3
|
||||
assert [i["id"] for i in body["items"]] == ["s1", "m2"] # first page of the added-desc order
|
||||
|
||||
|
||||
class TestRollup:
|
||||
def test_state_map_new_inprogress_watched(self, db):
|
||||
s = _seed(db)
|
||||
show, user = s["show"], s["user"]
|
||||
# e1 in-progress, e2 new -> in_progress
|
||||
assert rollup.state_map(db, user.id, [show.id]) == {show.id: "in_progress"}
|
||||
# empty input short-circuits
|
||||
assert rollup.state_map(db, user.id, []) == {}
|
||||
# a fresh user with no state -> new
|
||||
other = User(email="rollup-other@example.com")
|
||||
db.add(other)
|
||||
db.commit()
|
||||
assert rollup.state_map(db, other.id, [show.id]) == {show.id: "new"}
|
||||
# mark every episode watched for `user` -> watched
|
||||
db.execute(delete(PlexState).where(PlexState.user_id == user.id))
|
||||
for e in db.execute(select(PlexItem).where(PlexItem.kind == "episode")).scalars():
|
||||
db.add(PlexState(user_id=user.id, item_id=e.id, status="watched"))
|
||||
db.commit()
|
||||
assert rollup.state_map(db, user.id, [show.id]) == {show.id: "watched"}
|
||||
|
||||
|
||||
def _fake_paginate(by_type):
|
||||
def _p(_plex, _key, typ):
|
||||
return list(by_type.get(typ, []))
|
||||
return _p
|
||||
|
||||
|
||||
class TestSyncUpsertIdempotency:
|
||||
def test_movies_upsert_in_place_no_duplicates(self, db, monkeypatch):
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.commit()
|
||||
metas = [
|
||||
{"ratingKey": "100", "title": "First", "year": 2000, "addedAt": 1700000000},
|
||||
{"ratingKey": "101", "title": "Second", "year": 2001, "addedAt": 1700000001},
|
||||
]
|
||||
monkeypatch.setattr(sync, "_paginate", _fake_paginate({sync._TYPE_MOVIE: metas}))
|
||||
sync._sync_movies(db, object(), lib, {"movies": 0})
|
||||
db.commit()
|
||||
# second run with one title changed: same rows, updated in place (upsert, not re-insert)
|
||||
metas[0]["title"] = "First Renamed"
|
||||
sync._sync_movies(db, object(), lib, {"movies": 0})
|
||||
db.commit()
|
||||
rows = db.execute(
|
||||
select(PlexItem).where(PlexItem.library_id == lib.id, PlexItem.kind == "movie")
|
||||
).scalars().all()
|
||||
assert {r.rating_key for r in rows} == {"100", "101"} # no duplicate rows
|
||||
assert next(r for r in rows if r.rating_key == "100").title == "First Renamed"
|
||||
|
||||
def test_shows_upsert_links_episodes_and_is_idempotent(self, db, monkeypatch):
|
||||
lib = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||
db.add(lib)
|
||||
db.commit()
|
||||
by_type = {
|
||||
sync._TYPE_SHOW: [
|
||||
{"ratingKey": "s1", "title": "Show One", "childCount": 1, "addedAt": 1700000000}
|
||||
],
|
||||
sync._TYPE_SEASON: [
|
||||
{"ratingKey": "se1", "parentRatingKey": "s1", "title": "Season 1", "index": 1}
|
||||
],
|
||||
sync._TYPE_EPISODE: [
|
||||
{"ratingKey": "e1", "title": "Ep One", "grandparentRatingKey": "s1",
|
||||
"parentRatingKey": "se1", "parentIndex": 1, "index": 1, "addedAt": 1700000001}
|
||||
],
|
||||
}
|
||||
monkeypatch.setattr(sync, "_paginate", _fake_paginate(by_type))
|
||||
for _ in range(2): # run twice — must be idempotent
|
||||
sync._sync_shows(db, object(), lib, {"shows": 0, "seasons": 0, "episodes": 0})
|
||||
db.commit()
|
||||
shows = db.execute(select(PlexShow).where(PlexShow.library_id == lib.id)).scalars().all()
|
||||
assert {sh.rating_key for sh in shows} == {"s1"} # no duplicate show
|
||||
ep = db.execute(select(PlexItem).where(PlexItem.kind == "episode")).scalars().one()
|
||||
assert ep.rating_key == "e1"
|
||||
assert ep.show_id == shows[0].id # episode linked to its show
|
||||
assert (ep.season_number, ep.episode_number) == (1, 1)
|
||||
|
||||
|
||||
class TestFacetsWithData:
|
||||
"""The facets endpoint's wsq builder is the most complex converted query (genre/content-rating
|
||||
JSONB unnest + year/rating/duration bounds + the watch-state join). The existing smoke test only
|
||||
covers the null-genre guard; this locks the aggregated numbers with real data."""
|
||||
|
||||
def test_facet_aggregation(self, db, monkeypatch):
|
||||
movie_lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
show_lib = PlexLibrary(plex_key="2", title="Shows", kind="show", enabled=True)
|
||||
db.add_all([movie_lib, show_lib])
|
||||
db.flush()
|
||||
db.add_all([
|
||||
PlexItem(rating_key="m1", library_id=movie_lib.id, kind="movie", title="M1",
|
||||
genres=["Comedy", "Drama"], content_rating="PG", year=2000, rating=7.5,
|
||||
duration_s=6000),
|
||||
PlexItem(rating_key="m2", library_id=movie_lib.id, kind="movie", title="M2",
|
||||
genres=["Comedy"], content_rating="R", year=2010, rating=8.0, duration_s=7200),
|
||||
PlexShow(rating_key="s1", library_id=show_lib.id, title="S1",
|
||||
genres=["Drama"], content_rating="TV-14", year=2005, rating=9.0),
|
||||
])
|
||||
user = User(email="facetdata@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
|
||||
body = _get(db, user, monkeypatch, "/api/plex/facets?scope=both").json()
|
||||
assert {g["value"]: g["count"] for g in body["genres"]} == {"Comedy": 2, "Drama": 2}
|
||||
assert {c["value"]: c["count"] for c in body["content_ratings"]} == {
|
||||
"PG": 1, "R": 1, "TV-14": 1
|
||||
}
|
||||
assert (body["year_min"], body["year_max"]) == (2000, 2010)
|
||||
assert body["rating_max"] == 9.0
|
||||
# Duration bounds are movie-only.
|
||||
assert (body["duration_min"], body["duration_max"]) == (6000, 7200)
|
||||
|
||||
|
||||
class TestPlaylistAndCollectionEndpoints:
|
||||
"""Smoke coverage for the converted CRUD endpoints that had no tests: playlist create/list/detail/
|
||||
add/reorder/remove/delete, and the collections list. A 1.x->2.0 conversion slip in any of these
|
||||
(a lost .scalars(), a wrong .all(), a broken order_by) surfaces here instead of shipping green."""
|
||||
|
||||
def _movie(self, db, lib_id, rk, title):
|
||||
return PlexItem(rating_key=rk, library_id=lib_id, kind="movie", title=title)
|
||||
|
||||
def test_playlist_crud_roundtrip(self, db, monkeypatch):
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
db.add_all([self._movie(db, lib.id, "m1", "First"), self._movie(db, lib.id, "m2", "Second")])
|
||||
user = User(email="pl@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
|
||||
# create (seeded with m1) -> item_count 1
|
||||
created = _request(db, user, monkeypatch, "POST", "/api/plex/playlists",
|
||||
{"title": "Faves", "item_rating_key": "m1"})
|
||||
assert created.status_code == 200, created.text
|
||||
pid = created.json()["id"]
|
||||
assert created.json()["item_count"] == 1
|
||||
|
||||
# list -> the playlist with its count
|
||||
listed = _get(db, user, monkeypatch, "/api/plex/playlists").json()["playlists"]
|
||||
assert [(p["id"], p["item_count"]) for p in listed] == [(pid, 1)]
|
||||
|
||||
# add m2, then reorder to [m2, m1]
|
||||
add = _request(db, user, monkeypatch, "POST", f"/api/plex/playlists/{pid}/items",
|
||||
{"item_rating_key": "m2"})
|
||||
assert add.status_code == 200, add.text
|
||||
_request(db, user, monkeypatch, "PUT", f"/api/plex/playlists/{pid}/order",
|
||||
{"item_rating_keys": ["m2", "m1"]})
|
||||
detail = _get(db, user, monkeypatch, f"/api/plex/playlists/{pid}").json()
|
||||
assert [it["id"] for it in detail["items"]] == ["m2", "m1"] # reorder applied, in order
|
||||
|
||||
# remove m1 -> one item left
|
||||
rm = _request(db, user, monkeypatch, "DELETE", f"/api/plex/playlists/{pid}/items/m1")
|
||||
assert rm.status_code == 200, rm.text
|
||||
after = _get(db, user, monkeypatch, f"/api/plex/playlists/{pid}").json()
|
||||
assert [it["id"] for it in after["items"]] == ["m2"]
|
||||
|
||||
# delete the playlist -> gone from the list
|
||||
_request(db, user, monkeypatch, "DELETE", f"/api/plex/playlists/{pid}")
|
||||
assert _get(db, user, monkeypatch, "/api/plex/playlists").json()["playlists"] == []
|
||||
|
||||
def test_playlist_bulk_add_and_remove(self, db, monkeypatch):
|
||||
# Covers the bulk endpoints (add-bulk / remove-bulk) and pins the remove-bulk delete's
|
||||
# synchronize_session=False path, which the single add/remove roundtrip never exercises.
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
db.add_all([self._movie(db, lib.id, rk, rk.upper()) for rk in ("m1", "m2", "m3")])
|
||||
user = User(email="bulk@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
|
||||
created = _request(db, user, monkeypatch, "POST", "/api/plex/playlists", {"title": "Bulk"})
|
||||
pid = created.json()["id"]
|
||||
|
||||
added = _request(db, user, monkeypatch, "POST", f"/api/plex/playlists/{pid}/items/bulk",
|
||||
{"item_rating_keys": ["m1", "m2", "m3"]})
|
||||
assert added.status_code == 200, added.text
|
||||
assert added.json() == {"added": 3, "item_count": 3}
|
||||
|
||||
removed = _request(db, user, monkeypatch, "POST",
|
||||
f"/api/plex/playlists/{pid}/items/remove-bulk",
|
||||
{"item_rating_keys": ["m2"]})
|
||||
assert removed.status_code == 200, removed.text
|
||||
assert removed.json() == {"removed": 1, "item_count": 2}
|
||||
|
||||
detail = _get(db, user, monkeypatch, f"/api/plex/playlists/{pid}").json()
|
||||
assert [it["id"] for it in detail["items"]] == ["m1", "m3"] # input order preserved, m2 gone
|
||||
|
||||
def test_collections_list(self, db, monkeypatch):
|
||||
from app.models import PlexCollection
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
db.add_all([
|
||||
PlexCollection(rating_key="c1", library_id=lib.id, title="Marvel", child_count=12),
|
||||
PlexCollection(rating_key="c2", library_id=lib.id, title="Pixar", child_count=5),
|
||||
])
|
||||
user = User(email="col@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
|
||||
# unified scope (no library param) -> most-populated first
|
||||
body = _get(db, user, monkeypatch, "/api/plex/collections").json()
|
||||
assert [c["title"] for c in body["collections"]] == ["Marvel", "Pixar"]
|
||||
# name filter
|
||||
filtered = _get(db, user, monkeypatch, "/api/plex/collections?q=pix").json()
|
||||
assert [c["title"] for c in filtered["collections"]] == ["Pixar"]
|
||||
|
||||
def test_item_state_watched_then_new(self, db, monkeypatch):
|
||||
# No PlexLink -> the background Plex push is a no-op; only the local PlexState path runs, which
|
||||
# is what the conversion touched (the st lookup + the delete on ->new).
|
||||
lib = PlexLibrary(plex_key="1", title="Movies", kind="movie", enabled=True)
|
||||
db.add(lib)
|
||||
db.flush()
|
||||
db.add(PlexItem(rating_key="m1", library_id=lib.id, kind="movie", title="M1"))
|
||||
user = User(email="state@example.com")
|
||||
db.add(user)
|
||||
db.commit()
|
||||
|
||||
watched = _request(db, user, monkeypatch, "POST", "/api/plex/item/m1/state",
|
||||
{"status": "watched"})
|
||||
assert watched.status_code == 200, watched.text
|
||||
assert watched.json()["status"] == "watched"
|
||||
row = db.execute(select(PlexState).where(PlexState.user_id == user.id)).scalars().one()
|
||||
assert row.status == "watched"
|
||||
|
||||
# -> new deletes the state row
|
||||
back = _request(db, user, monkeypatch, "POST", "/api/plex/item/m1/state", {"status": "new"})
|
||||
assert back.status_code == 200, back.text
|
||||
assert db.execute(select(PlexState).where(PlexState.user_id == user.id)).scalars().first() is None
|
||||
@@ -55,3 +55,15 @@ def test_log_action_survives_caller_rollback(db):
|
||||
assert events[0].units == 0
|
||||
# ... and the caller's uncommitted user is gone.
|
||||
assert db.scalar(select(func.count()).select_from(User)) == 1
|
||||
|
||||
|
||||
def test_backfill_reserve_reached_tracks_the_boundary(db):
|
||||
# The shared yield gate for every backfill/explore/revalidation path: True once today's remaining
|
||||
# budget is at or below the backfill reserve. It reads in its OWN session, so it must reflect
|
||||
# record_usage's independently-committed spend — and the boundary is inclusive (<=). Defaults:
|
||||
# quota_daily_budget 9000, backfill_quota_reserve 2000 -> reached once 7000 units are used.
|
||||
assert quota.backfill_reserve_reached() is False # 0 used, remaining 9000
|
||||
quota.record_usage(6999) # remaining 2001, just above the reserve
|
||||
assert quota.backfill_reserve_reached() is False
|
||||
quota.record_usage(1) # remaining 2000, exactly at the reserve
|
||||
assert quota.backfill_reserve_reached() is True # boundary is <=, so the reserve is reached
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Crypto primitives — a guard for the cryptography (Fernet) + argon2 dependency bumps.
|
||||
|
||||
Added with the cryptography 46->50 CVE bump: encrypt/decrypt/decrypt_optional had NO test, and they are
|
||||
the token-at-rest path (Google/API/SMTP/Plex secrets in system_config, OAuth refresh tokens). A fresh
|
||||
Fernet key is monkeypatched in so the test doesn't depend on TOKEN_ENCRYPTION_KEY being set in the env.
|
||||
Pure-logic (no DB).
|
||||
"""
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
from app import security
|
||||
|
||||
|
||||
def test_fernet_encrypt_decrypt_roundtrip(monkeypatch):
|
||||
monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key()))
|
||||
assert security.decrypt(security.encrypt("hunter2")) == "hunter2"
|
||||
assert security.encrypt(None) is None
|
||||
assert security.decrypt(None) is None
|
||||
|
||||
|
||||
def test_decrypt_optional_degrades_instead_of_raising(monkeypatch):
|
||||
# Garbage / absent input -> None (never raises).
|
||||
monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key()))
|
||||
assert security.decrypt_optional("not-a-valid-token") is None
|
||||
assert security.decrypt_optional(None) is None
|
||||
# And a ciphertext written under a DIFFERENT key (the TOKEN_ENCRYPTION_KEY-mismatch case that B-core-2
|
||||
# made non-fatal) also degrades to None rather than raising InvalidToken.
|
||||
ciphertext = security.encrypt("secret")
|
||||
monkeypatch.setattr(security, "_fernet", Fernet(Fernet.generate_key())) # rotate the key
|
||||
assert security.decrypt_optional(ciphertext) is None
|
||||
|
||||
|
||||
def test_password_hash_and_verify_roundtrip():
|
||||
hashed = security.hash_password("correct horse battery")
|
||||
assert security.verify_password("correct horse battery", hashed) is True
|
||||
assert security.verify_password("wrong password", hashed) is False
|
||||
assert security.verify_password("anything", None) is False
|
||||
@@ -137,3 +137,38 @@ class TestSafeAbsPath:
|
||||
|
||||
def test_returns_none_for_a_nonexistent_file_inside_the_root(self, tmp_path: Path):
|
||||
assert safe_abs_path(str(tmp_path), "does/not/exist.mp4") is None
|
||||
|
||||
|
||||
def test_nfo_uniqueid_type_reflects_the_source():
|
||||
# The .nfo <uniqueid type=...> now follows the source (via SourceAdapter.nfo_uniqueid_type),
|
||||
# instead of the old hardcoded "youtube" that mislabelled every url-source download.
|
||||
from app.downloads.storage import _nfo_xml
|
||||
|
||||
base = dict(video_id="v1", title="t", uploader="c", upload_date=None)
|
||||
# An un-set caller gets the source-neutral "generic" default, never a false "youtube".
|
||||
assert '<uniqueid type="generic"' in _nfo_xml(MediaMeta(**base))
|
||||
assert '<uniqueid type="facebook"' in _nfo_xml(MediaMeta(**base, uniqueid_type="facebook"))
|
||||
assert '<uniqueid type="youtube"' in _nfo_xml(MediaMeta(**base, uniqueid_type="youtube"))
|
||||
|
||||
|
||||
def test_sidecars_keep_the_full_basename_when_the_title_has_a_dot(tmp_path: Path):
|
||||
# A title ending in a period ("Sonka Házilag.") puts a dot in the media slug; the .nfo/.jpg
|
||||
# sidecars must sit on the SAME (full) basename, not be truncated at that dot (which orphaned
|
||||
# them from the media so Plex/Kodi never matched them).
|
||||
from app.downloads.storage import write_sidecars
|
||||
|
||||
stem = "Chan_-_2026-08-12_-_Sonka_Hazilag._Villam_konyha_abc_a5"
|
||||
rel = f"Chan/Season_2026/{stem}.mp4"
|
||||
media = tmp_path / rel
|
||||
media.parent.mkdir(parents=True)
|
||||
media.write_text("x")
|
||||
thumb = tmp_path / "t.jpg"
|
||||
thumb.write_bytes(b"\xff\xd8\xff")
|
||||
meta = MediaMeta(video_id="abc", title="Sonka Házilag. | Villám konyha", uploader="Chan",
|
||||
upload_date=None)
|
||||
|
||||
assert write_sidecars(str(tmp_path), rel, meta, thumb) is True
|
||||
names = sorted(p.name for p in media.parent.iterdir())
|
||||
assert f"{stem}.nfo" in names, names
|
||||
assert f"{stem}.jpg" in names, names
|
||||
assert "Chan_-_2026-08-12_-_Sonka_Hazilag.nfo" not in names # the old truncated orphan
|
||||
|
||||
@@ -88,7 +88,7 @@ def test_no_progress_stall_kills_and_fails_the_job(monkeypatch, rec):
|
||||
|
||||
|
||||
def test_route_pause_kills_but_keeps_the_route_status(monkeypatch, rec):
|
||||
# A route flipped the job to 'paused'/'cancelled'. Kill + free the asset, but do NOT overwrite the
|
||||
# A route flipped the job to 'paused'. Kill + free the asset, but do NOT overwrite the
|
||||
# status with an error and do NOT requeue.
|
||||
_spawn(monkeypatch, _FakeProc(["timeout"]))
|
||||
_liveness(monkeypatch, ("paused", 1.0))
|
||||
@@ -98,6 +98,19 @@ def test_route_pause_kills_but_keeps_the_route_status(monkeypatch, rec):
|
||||
assert "set_job" not in names and "requeue" not in names
|
||||
|
||||
|
||||
def test_route_cancel_kills_but_keeps_the_route_status(monkeypatch, rec):
|
||||
# B-sub-1 regression: the cancel ROUTE writes the literal 'canceled' (models.py), but the supervisor
|
||||
# used to compare against 'cancelled' — so this branch was DEAD and a canceled job's child ran on to
|
||||
# completion, overwriting 'canceled' with 'done' and double-releasing its asset. The supervisor MUST
|
||||
# treat the route's actual literal as a stop: kill + free the asset, no error write, no requeue.
|
||||
_spawn(monkeypatch, _FakeProc(["timeout"]))
|
||||
_liveness(monkeypatch, ("canceled", 1.0))
|
||||
worker._run_supervised(1, 99)
|
||||
names = _names(rec)
|
||||
assert "stop_child" in names and "reset_asset" in names
|
||||
assert "set_job" not in names and "requeue" not in names
|
||||
|
||||
|
||||
def test_wall_clock_cap_kills_even_while_progressing(monkeypatch, rec):
|
||||
monkeypatch.setattr(worker.settings, "download_wall_clock_max_seconds", 0)
|
||||
_spawn(monkeypatch, _FakeProc(["timeout"]))
|
||||
|
||||
@@ -41,7 +41,10 @@ The installer asks for the **public URL** where the instance will be reached (ju
|
||||
`http://localhost:8080` to try it locally). It then:
|
||||
|
||||
- generates the secrets it needs (`SECRET_KEY`, `TOKEN_ENCRYPTION_KEY`, a database password) into a
|
||||
local `.env` file — keep that file private,
|
||||
local `.env` file — keep that file private. **Do not change `TOKEN_ENCRYPTION_KEY` once the app has
|
||||
stored any secret** (Google credentials, API keys, SMTP password, Plex token): those values are
|
||||
encrypted with it, and a changed key makes them unreadable. If it is ever lost or rotated, clear the
|
||||
affected rows from the `system_config` table and re-enter the secrets in the admin Configuration page,
|
||||
- pulls the image and starts the app + database,
|
||||
- prints the **setup wizard URL**, which looks like `…/setup?token=…`.
|
||||
|
||||
|
||||
@@ -351,7 +351,13 @@ export default function App() {
|
||||
/>
|
||||
</Suspense>
|
||||
)}
|
||||
<div className="relative flex-1 min-w-0 flex flex-col">
|
||||
{/* `@container/appmain`: the floating header + its search box adapt to THIS column's width
|
||||
(which shrinks when the filter sidebar is open), not the viewport — a viewport breakpoint
|
||||
can't see the sidebar-driven narrowing (E5 S7). NB: `container-type: inline-size` implies
|
||||
`contain: layout`, making this column a containing block for `position: fixed` descendants
|
||||
too — safe today because floating-ui menus render via FloatingPortal (to document.body) and
|
||||
the toasts below are absolute; keep any new fixed overlay inside here body-portalled. */}
|
||||
<div className="@container/appmain relative flex-1 min-w-0 flex flex-col">
|
||||
{channelView ? (
|
||||
<PageShell scrollKey={`channel:${channelView.id}`}>
|
||||
<Suspense fallback={pageFallback}>
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// Contract of the reusable ⋯ overflow menu. Labels/handlers are plain props (no i18n), so we assert
|
||||
// the open/select/close + disabled-item behaviour. The menu content only mounts while open, and the
|
||||
// AnchoredPopover platform is floating-ui based, so we stub the two observers autoUpdate wires and
|
||||
// wrap in a FloatingTree (its real-app parent) so nested-layer dispatch has a node to register.
|
||||
import { describe, it, expect, vi, afterEach, beforeAll } from "vitest";
|
||||
import { render, screen, fireEvent, cleanup } from "@testing-library/react";
|
||||
import { FloatingTree } from "@floating-ui/react";
|
||||
import ActionMenu, { type ActionMenuItem } from "./ActionMenu";
|
||||
|
||||
beforeAll(() => {
|
||||
// floating-ui's autoUpdate observes with ResizeObserver + IntersectionObserver (layoutShift); jsdom
|
||||
// ships neither, and the floating element mounts (so they run) the moment the menu opens.
|
||||
class Stub {
|
||||
observe() {}
|
||||
unobserve() {}
|
||||
disconnect() {}
|
||||
takeRecords() {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
globalThis.ResizeObserver ??= Stub as unknown as typeof ResizeObserver;
|
||||
globalThis.IntersectionObserver ??= Stub as unknown as typeof IntersectionObserver;
|
||||
});
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
const renderMenu = (items: ActionMenuItem[]) =>
|
||||
render(
|
||||
<FloatingTree>
|
||||
<ActionMenu ariaLabel="Catalog actions" items={items} />
|
||||
</FloatingTree>,
|
||||
);
|
||||
|
||||
describe("ActionMenu", () => {
|
||||
it("renders a labelled trigger and keeps items unmounted until opened", () => {
|
||||
renderMenu([{ id: "sync", label: "Sync", onSelect: vi.fn() }]);
|
||||
expect(screen.getByRole("button", { name: "Catalog actions" })).toBeTruthy();
|
||||
expect(screen.queryByRole("menuitem")).toBeNull();
|
||||
});
|
||||
|
||||
// floating-ui never computes a position in jsdom, so the surface stays visibility:hidden; the items
|
||||
// are in the DOM but the a11y tree treats them as hidden — hence `{ hidden: true }` on role queries.
|
||||
it("opens on click and shows one menuitem per item", () => {
|
||||
renderMenu([
|
||||
{ id: "sync", label: "Sync", onSelect: vi.fn() },
|
||||
{ id: "backfill", label: "Backfill", onSelect: vi.fn() },
|
||||
]);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Catalog actions" }));
|
||||
expect(screen.getAllByRole("menuitem", { hidden: true }).map((b) => b.textContent)).toEqual([
|
||||
"Sync",
|
||||
"Backfill",
|
||||
]);
|
||||
});
|
||||
|
||||
it("selecting an item fires its onSelect and closes the menu", () => {
|
||||
const onSelect = vi.fn();
|
||||
renderMenu([{ id: "sync", label: "Sync", onSelect }]);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Catalog actions" }));
|
||||
fireEvent.click(screen.getByText("Sync"));
|
||||
expect(onSelect).toHaveBeenCalledTimes(1);
|
||||
expect(screen.queryByRole("menuitem", { hidden: true })).toBeNull();
|
||||
});
|
||||
|
||||
it("a disabled item is inert — rendered disabled and never fires onSelect", () => {
|
||||
const onSelect = vi.fn();
|
||||
renderMenu([{ id: "sync", label: "Sync", onSelect, disabled: true }]);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Catalog actions" }));
|
||||
const item = screen.getByText("Sync").closest("button")!;
|
||||
expect((item as HTMLButtonElement).disabled).toBe(true);
|
||||
fireEvent.click(item);
|
||||
expect(onSelect).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("skips a disabled item in roving keyboard nav", () => {
|
||||
// items[0] disabled, items[1] + items[2] enabled. Roving keyboard focus (the `tabindex=0` item,
|
||||
// which tracks useListNavigation's activeIndex) must NEVER land on the disabled index-0 — arrowing
|
||||
// skips straight over it. The skip comes for free: PopoverItem renders a native `disabled` button
|
||||
// and useListNavigation reads that DOM state (no explicit disabledIndices needed). Without it,
|
||||
// ArrowUp/Down would strand roving focus on the inert index-0 button. (floating-ui wires this on
|
||||
// real key nav, not on open, in jsdom — so we drive it with arrow keys.)
|
||||
renderMenu([
|
||||
{ id: "sync", label: "Sync", onSelect: vi.fn(), disabled: true },
|
||||
{ id: "backfill", label: "Backfill", onSelect: vi.fn() },
|
||||
{ id: "reset", label: "Reset", onSelect: vi.fn() },
|
||||
]);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Catalog actions" }));
|
||||
const [disabled, backfill, reset] = screen.getAllByRole("menuitem", { hidden: true }) as [
|
||||
HTMLElement,
|
||||
HTMLElement,
|
||||
HTMLElement,
|
||||
];
|
||||
const menu = screen.getByRole("menu", { hidden: true });
|
||||
const rover = () =>
|
||||
[disabled, backfill, reset].find((el) => el.getAttribute("tabindex") === "0")?.textContent;
|
||||
|
||||
// The disabled item never becomes the roving target, whichever way we arrow.
|
||||
fireEvent.keyDown(menu, { key: "ArrowDown" }); // skips disabled[0] -> first enabled
|
||||
expect(rover()).toBe("Backfill");
|
||||
expect(disabled.getAttribute("tabindex")).toBe("-1");
|
||||
|
||||
fireEvent.keyDown(menu, { key: "ArrowDown" });
|
||||
expect(rover()).toBe("Reset");
|
||||
|
||||
fireEvent.keyDown(menu, { key: "ArrowUp" }); // back to Backfill
|
||||
expect(rover()).toBe("Backfill");
|
||||
fireEvent.keyDown(menu, { key: "ArrowUp" }); // wraps over disabled[0] to the last item
|
||||
expect(rover()).toBe("Reset");
|
||||
expect(disabled.getAttribute("tabindex")).toBe("-1"); // still never the disabled item
|
||||
});
|
||||
|
||||
it("honours a custom trigger node", () => {
|
||||
renderMenu([{ id: "sync", label: "Sync", onSelect: vi.fn() }]);
|
||||
cleanup();
|
||||
render(
|
||||
<FloatingTree>
|
||||
<ActionMenu
|
||||
ariaLabel="More"
|
||||
items={[{ id: "x", label: "X", onSelect: vi.fn() }]}
|
||||
trigger={<span data-testid="custom-trigger">•••</span>}
|
||||
/>
|
||||
</FloatingTree>,
|
||||
);
|
||||
expect(screen.getByTestId("custom-trigger")).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,78 @@
|
||||
import { useState, type ReactNode } from "react";
|
||||
import { MoreHorizontal } from "lucide-react";
|
||||
import { PopoverItem, PopoverSurface, useAnchoredPopover } from "./AnchoredPopover";
|
||||
|
||||
// A reusable "⋯ more actions" overflow menu: a compact trigger that opens a list of one-shot actions.
|
||||
// Built on the R8 AnchoredPopover platform (arrows/typeahead/Escape/anchor-flip), same as ViewSwitcher
|
||||
// and LanguageSwitcher. Use it to keep occasional actions out of a crowded toolbar. The items are
|
||||
// fire-and-close (`onSelect`); a `disabled` item (e.g. a mutation already in flight) is inert.
|
||||
|
||||
export interface ActionMenuItem {
|
||||
id: string;
|
||||
label: string;
|
||||
icon?: ReactNode; // an element (so a caller can swap in a spinning icon while pending)
|
||||
onSelect: () => void;
|
||||
disabled?: boolean;
|
||||
title?: string; // optional native tooltip (carries a longer hint the label alone can't)
|
||||
}
|
||||
|
||||
export default function ActionMenu({
|
||||
items,
|
||||
ariaLabel,
|
||||
className,
|
||||
trigger,
|
||||
}: {
|
||||
items: readonly ActionMenuItem[];
|
||||
/** Accessible name for the trigger (e.g. "Catalog actions"). */
|
||||
ariaLabel: string;
|
||||
/** Trigger className override; omit for the default glass ⋯ icon button. */
|
||||
className?: string;
|
||||
/** Custom trigger content; omit for the default ⋯ icon. */
|
||||
trigger?: ReactNode;
|
||||
}) {
|
||||
const [open, setOpen] = useState(false);
|
||||
// A disabled item (a mutation already in flight) is skipped by roving keyboard nav automatically:
|
||||
// PopoverItem renders it as a native `disabled` button, which useListNavigation reads and never
|
||||
// focuses. No explicit disabledIndices needed. See ActionMenu.dom.test.tsx for the guard.
|
||||
const pop = useAnchoredPopover({ open, onOpenChange: setOpen, placement: "bottom-end" });
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
ref={pop.refs.setReference}
|
||||
{...pop.getReferenceProps()}
|
||||
aria-label={ariaLabel}
|
||||
title={ariaLabel}
|
||||
className={
|
||||
className ??
|
||||
"glass-card glass-hover flex items-center justify-center rounded-xl p-2 text-sm transition"
|
||||
}
|
||||
>
|
||||
{trigger ?? <MoreHorizontal className="h-5 w-5" aria-hidden="true" />}
|
||||
</button>
|
||||
<PopoverSurface
|
||||
popover={pop}
|
||||
ariaLabel={ariaLabel}
|
||||
className="glass-menu min-w-52 rounded-xl p-1.5 animate-[popIn_0.16s_ease]"
|
||||
>
|
||||
{items.map((it) => (
|
||||
<PopoverItem
|
||||
key={it.id}
|
||||
role="menuitem"
|
||||
disabled={it.disabled}
|
||||
label={it.label}
|
||||
title={it.title}
|
||||
onSelect={() => {
|
||||
it.onSelect();
|
||||
setOpen(false);
|
||||
}}
|
||||
className="flex w-full items-center gap-2.5 rounded-lg px-2 py-1.5 text-left text-sm outline-hidden transition disabled:opacity-50"
|
||||
>
|
||||
{it.icon && <span className="shrink-0 text-muted">{it.icon}</span>}
|
||||
<span className="flex-1 truncate">{it.label}</span>
|
||||
</PopoverItem>
|
||||
))}
|
||||
</PopoverSurface>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -175,6 +175,10 @@ export function useAnchoredPopover(opts: AnchoredPopoverOptions) {
|
||||
setHoverIndex(null);
|
||||
},
|
||||
selectedIndex: opts.selectedIndex ?? null,
|
||||
// Disabled items are skipped automatically: with no `disabledIndices`, useListNavigation reads
|
||||
// each item's DOM `disabled`/`aria-disabled` (PopoverItem always renders native `disabled`), so a
|
||||
// pending/inert item is never a roving target. (Verified against @floating-ui/react 0.27
|
||||
// `isListIndexDisabled`.) No explicit index list needed.
|
||||
loop: true,
|
||||
// Focus the first item on open for BOTH click and keyboard (the default 'auto' skips pointer-opens,
|
||||
// which strands focus on the menu container — arrows then can't enter the roving list at all).
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useEffect } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { qk } from "../lib/queryKeys";
|
||||
import { UserPlus } from "lucide-react";
|
||||
import { api, type DiscoveredChannel } from "../lib/api";
|
||||
import { accountKey, LS, readAccount, writeAccount } from "../lib/storage";
|
||||
import { accountKey, LS } from "../lib/storage";
|
||||
import { formatCountOrDash } from "../lib/format";
|
||||
import { parseSortState, sortRows, type SortState } from "../lib/columnSort";
|
||||
import { sortRows } from "../lib/columnSort";
|
||||
import { useSortState } from "../lib/useSortState";
|
||||
import { useCardPager } from "../lib/useCardPager";
|
||||
import { subsColumn } from "./channelColumns";
|
||||
import { notify } from "../lib/notifications";
|
||||
@@ -44,13 +45,7 @@ export default function ChannelDiscovery({
|
||||
// The table/cards density is shared with the Subscriptions tab (ChannelsProvider). The sort is
|
||||
// this tab's own (its columns differ), lifted out of the table so the card layout shares it.
|
||||
const layout = useChannelLayout();
|
||||
const [sort, setSortState] = useState<SortState>(() =>
|
||||
parseSortState(readAccount(LS.channelDiscoverySort, null)),
|
||||
);
|
||||
const setSort = (s: SortState) => {
|
||||
setSortState(s);
|
||||
writeAccount(LS.channelDiscoverySort, s);
|
||||
};
|
||||
const [sort, setSort] = useSortState(LS.channelDiscoverySort);
|
||||
|
||||
const query = useQuery({
|
||||
queryKey: qk.discoveredChannels(),
|
||||
|
||||
@@ -41,12 +41,16 @@ export default function ChannelLink({
|
||||
onMouseDown={(e) => {
|
||||
if (e.button === 1) e.preventDefault(); // suppress middle-click autoscroll
|
||||
}}
|
||||
// title = the full name, so a truncated fluid-column name is still readable on hover.
|
||||
title={name}
|
||||
className="text-sm font-medium truncate hover:text-accent text-left min-w-0"
|
||||
>
|
||||
{name}
|
||||
</button>
|
||||
) : (
|
||||
<span className="text-sm font-medium truncate min-w-0">{name}</span>
|
||||
<span title={name} className="text-sm font-medium truncate min-w-0">
|
||||
{name}
|
||||
</span>
|
||||
)}
|
||||
<Tooltip hint={t("channels.row.openOnYouTube")}>
|
||||
<a
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { useLiveQuery } from "../lib/useLiveQuery";
|
||||
import { channelDetailPollMs } from "../lib/channelDetailPoll";
|
||||
import { qk } from "../lib/queryKeys";
|
||||
import { useNavigationActions } from "./NavigationProvider";
|
||||
import { PriorityStepper } from "./PriorityStepper";
|
||||
@@ -17,6 +19,7 @@ import {
|
||||
SlidersHorizontal,
|
||||
} from "lucide-react";
|
||||
import Avatar from "./Avatar";
|
||||
import Tabs from "./Tabs";
|
||||
import Feed from "./Feed";
|
||||
import ChannelAboutContent from "./ChannelAbout";
|
||||
import { PageToolbar, PageToolbarSlot } from "./PageShell";
|
||||
@@ -78,9 +81,12 @@ export default function ChannelPage({
|
||||
const MAX_BANNER_RETRIES = 3;
|
||||
const [bannerAttempt, setBannerAttempt] = useState(0);
|
||||
|
||||
const detail = useQuery({
|
||||
queryKey: qk.channel(channelId),
|
||||
queryFn: () => api.channelDetail(channelId),
|
||||
// A LIVE query: while THIS user's full-history backfill is running, poll so the stored count (and
|
||||
// the N/S/L breakdown) climb on their own — the immediate backfill fills it in seconds, and the
|
||||
// scheduler tops up any tail on its ~10-minute ticks — then stop polling once the back-catalogue is
|
||||
// in. Same queryKey as before, so mutation invalidations still hit.
|
||||
const detail = useLiveQuery(qk.channel(channelId), () => api.channelDetail(channelId), {
|
||||
intervalMs: channelDetailPollMs,
|
||||
});
|
||||
const ch = detail.data;
|
||||
useEffect(() => setBannerAttempt(0), [ch?.banner_url]); // fresh channel/banner → try again
|
||||
@@ -263,13 +269,17 @@ export default function ChannelPage({
|
||||
ch?.total_view_count != null
|
||||
? t("channel.totalViews", { formatted: formatViews(ch.total_view_count) })
|
||||
: null,
|
||||
// Per-type breakdown (R14 moved this off the manager table's N/S/L column). Only meaningful
|
||||
// once we actually hold some of the channel's videos.
|
||||
ch && ch.stored_videos > 0
|
||||
? t("channel.videoTypes", {
|
||||
normal: ch.count_normal,
|
||||
short: ch.count_short,
|
||||
live: ch.count_live,
|
||||
})
|
||||
: null,
|
||||
joined ? t("channel.joined", { date: joined }) : null,
|
||||
].filter(Boolean) as string[];
|
||||
const tabClass = (active: boolean) =>
|
||||
active
|
||||
? "pb-2 text-sm font-medium border-b-2 border-fg text-fg"
|
||||
: "pb-2 text-sm text-muted hover:text-fg border-b-2 border-transparent";
|
||||
|
||||
// The channel page owns no scroller of its own (PageShell provides the one app scroller). Its
|
||||
// whole chrome is fixed — wrapped in <PageToolbar> it portals into the shell's fixed band, so the
|
||||
// banner/identity/tabs/toolbar stay put and only the video grid scrolls.
|
||||
@@ -370,14 +380,17 @@ export default function ChannelPage({
|
||||
{/* Tabs — the channel actions ride along here rather than off in the header's top-right
|
||||
corner: there are only ever two tabs, so there's room, and it keeps the things you click
|
||||
within a few pixels of each other instead of across the page. */}
|
||||
<div className="flex items-center gap-4 mt-3 border-b border-border">
|
||||
<button onClick={() => setTab("videos")} className={tabClass(tab === "videos")}>
|
||||
{t("channel.tabVideos")}
|
||||
</button>
|
||||
<button onClick={() => setTab("about")} className={tabClass(tab === "about")}>
|
||||
{t("channel.tabAbout")}
|
||||
</button>
|
||||
<div className="ml-2 flex items-center gap-2 pb-1.5">
|
||||
<div className="flex items-center gap-2 mt-3 border-b border-border pb-2">
|
||||
<Tabs
|
||||
className="flex gap-1"
|
||||
active={tab}
|
||||
onChange={(id) => setTab(id as "videos" | "about")}
|
||||
tabs={[
|
||||
{ id: "videos", label: t("channel.tabVideos") },
|
||||
{ id: "about", label: t("channel.tabAbout") },
|
||||
]}
|
||||
/>
|
||||
<div className="ml-2 flex items-center gap-2">
|
||||
<a
|
||||
href={ytUrl}
|
||||
target="_blank"
|
||||
@@ -409,13 +422,18 @@ export default function ChannelPage({
|
||||
</div>
|
||||
)}
|
||||
{/* Only the recent subset is stored → offer to pull the channel's full back-catalogue.
|
||||
Once queued (deep_requested) it shows as loading; hidden once the backfill is done. */}
|
||||
Once queued (deep_requested) it shows as loading; hidden once the backfill is done.
|
||||
While loading, the label carries the live stored/total count (the detail poll refreshes
|
||||
it) so a large channel's paced tail reads as visible progress rather than a stuck
|
||||
spinner. */}
|
||||
{ch?.subscribed && !ch.backfill_done && !me.is_demo && (
|
||||
<button
|
||||
onClick={() => !ch.deep_requested && deepBackfill.mutate()}
|
||||
disabled={ch.deep_requested || deepBackfill.isPending}
|
||||
title={
|
||||
ch.deep_requested ? t("channel.fullHistoryLoading") : t("channel.fullHistory")
|
||||
ch.deep_requested
|
||||
? t("channel.fullHistoryLoadingHint")
|
||||
: t("channel.fullHistory")
|
||||
}
|
||||
className="inline-flex items-center gap-1.5 px-3 py-2 rounded-lg text-sm border border-border text-muted hover:text-accent hover:border-accent disabled:opacity-60 disabled:hover:text-muted disabled:hover:border-border transition"
|
||||
>
|
||||
@@ -425,7 +443,14 @@ export default function ChannelPage({
|
||||
<History className="w-4 h-4" />
|
||||
)}
|
||||
<span className="hidden sm:inline whitespace-nowrap">
|
||||
{ch.deep_requested ? t("channel.fullHistoryLoading") : t("channel.fullHistory")}
|
||||
{!ch.deep_requested
|
||||
? t("channel.fullHistory")
|
||||
: ch.video_count != null && ch.stored_videos < ch.video_count
|
||||
? t("channel.fullHistoryProgress", {
|
||||
stored: ch.stored_videos,
|
||||
total: ch.video_count,
|
||||
})
|
||||
: t("channel.fullHistoryLoading")}
|
||||
</span>
|
||||
</button>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// Contract of the catalog-stats hover chip: the compact trigger, the rows it reveals on open (value
|
||||
// formatting + the conditional "left" row), and — the point of the R13-S3 review fix — that each row
|
||||
// still carries its per-stat explanation as a `title`, so the guidance the old inline stats showed is
|
||||
// not lost. i18n is mocked to a key-passthrough so assertions read the stable keys, not translations.
|
||||
// The AnchoredPopover platform is floating-ui based (hover mode), so we stub the observers autoUpdate
|
||||
// wires and wrap in a FloatingTree, and open via focus (hover mode enables useFocus).
|
||||
import { describe, it, expect, afterEach, beforeAll, vi } from "vitest";
|
||||
import { render, screen, fireEvent, cleanup } from "@testing-library/react";
|
||||
import { FloatingTree } from "@floating-ui/react";
|
||||
|
||||
// Key-passthrough t so assertions read stable keys — but keep the rest of react-i18next intact
|
||||
// (i18n/index.ts calls i18n.use(initReactI18next) at import time).
|
||||
vi.mock("react-i18next", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("react-i18next")>();
|
||||
return { ...actual, useTranslation: () => ({ t: (key: string) => key }) };
|
||||
});
|
||||
|
||||
import ChannelStatsPopover from "./ChannelStatsPopover";
|
||||
import { type MyStatus } from "../lib/api";
|
||||
import { formatEta } from "../lib/format";
|
||||
|
||||
beforeAll(() => {
|
||||
class Stub {
|
||||
observe() {}
|
||||
unobserve() {}
|
||||
disconnect() {}
|
||||
takeRecords() {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
globalThis.ResizeObserver ??= Stub as unknown as typeof ResizeObserver;
|
||||
globalThis.IntersectionObserver ??= Stub as unknown as typeof IntersectionObserver;
|
||||
});
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
const status = (over: Partial<MyStatus> = {}): MyStatus =>
|
||||
({
|
||||
channels_total: 10,
|
||||
channels_recent_synced: 3,
|
||||
channels_deep_done: 1,
|
||||
channels_deep_requested: 4,
|
||||
deep_pending_count: 0,
|
||||
deep_eta_seconds: 120,
|
||||
my_videos: 1234,
|
||||
quota_remaining_today: 5000,
|
||||
...over,
|
||||
}) as MyStatus;
|
||||
|
||||
const open = (s: MyStatus) => {
|
||||
render(
|
||||
<FloatingTree>
|
||||
<ChannelStatsPopover status={s} />
|
||||
</FloatingTree>,
|
||||
);
|
||||
fireEvent.focus(screen.getByRole("button", { name: "channels.stats.label" }));
|
||||
};
|
||||
|
||||
describe("ChannelStatsPopover", () => {
|
||||
it("renders a compact Stats trigger, panel closed until opened", () => {
|
||||
render(
|
||||
<FloatingTree>
|
||||
<ChannelStatsPopover status={status()} />
|
||||
</FloatingTree>,
|
||||
);
|
||||
expect(screen.getByRole("button", { name: "channels.stats.label" })).toBeTruthy();
|
||||
expect(screen.queryByText("channels.stats.channels")).toBeNull();
|
||||
});
|
||||
|
||||
it("reveals one row per stat with formatted values", () => {
|
||||
open(status());
|
||||
expect(screen.getByText("channels.stats.recentSynced")).toBeTruthy();
|
||||
expect(screen.getByText("3/10")).toBeTruthy(); // recent-synced ratio
|
||||
expect(screen.getByText("1/4")).toBeTruthy(); // full-history ratio
|
||||
expect(screen.getByText((1234).toLocaleString())).toBeTruthy(); // my_videos, locale-agnostic
|
||||
});
|
||||
|
||||
it("shows the 'left' ETA row only while a backfill is pending, with the formatted ETA value", () => {
|
||||
open(status({ deep_pending_count: 0 }));
|
||||
expect(screen.queryByText("channels.stats.left")).toBeNull();
|
||||
cleanup();
|
||||
open(status({ deep_pending_count: 2, deep_eta_seconds: 120 }));
|
||||
expect(screen.getByText("channels.stats.left")).toBeTruthy();
|
||||
// The value is the FORMATTED ETA (via formatEta), not raw seconds — asserting it guards the
|
||||
// formatEta wiring, which is untested anywhere else. Using the same formatEta keeps this
|
||||
// locale/impl-agnostic (expected === what the component renders).
|
||||
expect(screen.getByText(formatEta(120))).toBeTruthy();
|
||||
expect(screen.queryByText("120")).toBeNull(); // never the raw seconds
|
||||
});
|
||||
|
||||
it("keeps each stat's explanation as a per-row title (hint not lost)", () => {
|
||||
open(status());
|
||||
const row = screen.getByText("channels.stats.channels").closest("div");
|
||||
expect(row?.getAttribute("title")).toBe("channels.stats.channelsHint");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,107 @@
|
||||
import { useState } from "react";
|
||||
import {
|
||||
BarChart3,
|
||||
Clock,
|
||||
Film,
|
||||
History,
|
||||
RefreshCw,
|
||||
Users,
|
||||
Zap,
|
||||
type LucideIcon,
|
||||
} from "lucide-react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { type MyStatus } from "../lib/api";
|
||||
import { formatEta } from "../lib/format";
|
||||
import { PopoverSurface, useAnchoredPopover } from "./AnchoredPopover";
|
||||
|
||||
// The channel-manager catalog stats, collapsed from a crowded inline row into a compact "Stats" chip
|
||||
// that reveals the numbers on hover (reuses the AnchoredPopover hover mode). One row per stat, each
|
||||
// with a leading icon. Kept in the manager's own toolbar (the header's SyncStatus is global; these
|
||||
// are channel-catalog figures).
|
||||
export default function ChannelStatsPopover({ status }: { status: MyStatus }) {
|
||||
const { t } = useTranslation();
|
||||
const [open, setOpen] = useState(false);
|
||||
const pop = useAnchoredPopover({
|
||||
open,
|
||||
onOpenChange: setOpen,
|
||||
mode: "hover",
|
||||
placement: "bottom-end",
|
||||
offset: 8,
|
||||
});
|
||||
|
||||
// Each row keeps the per-stat explanation the old inline stats carried (as a native `title` on the
|
||||
// row), so hovering a figure still tells the user what it counts — the guidance is not lost.
|
||||
const rows: { icon: LucideIcon; label: string; value: string | number; hint: string }[] = [
|
||||
{
|
||||
icon: Users,
|
||||
label: t("channels.stats.channels"),
|
||||
value: status.channels_total,
|
||||
hint: t("channels.stats.channelsHint"),
|
||||
},
|
||||
{
|
||||
icon: RefreshCw,
|
||||
label: t("channels.stats.recentSynced"),
|
||||
value: `${status.channels_recent_synced}/${status.channels_total}`,
|
||||
hint: t("channels.stats.recentSyncedHint"),
|
||||
},
|
||||
{
|
||||
icon: History,
|
||||
label: t("channels.stats.fullHistory"),
|
||||
value: `${status.channels_deep_done}/${status.channels_deep_requested}`,
|
||||
hint: t("channels.stats.fullHistoryHint"),
|
||||
},
|
||||
...(status.deep_pending_count > 0
|
||||
? [
|
||||
{
|
||||
icon: Clock,
|
||||
label: t("channels.stats.left"),
|
||||
value: formatEta(status.deep_eta_seconds),
|
||||
hint: t("channels.stats.leftHint", { count: status.deep_pending_count }),
|
||||
},
|
||||
]
|
||||
: []),
|
||||
{
|
||||
icon: Film,
|
||||
label: t("channels.stats.myVideos"),
|
||||
value: status.my_videos.toLocaleString(),
|
||||
hint: t("channels.stats.myVideosHint"),
|
||||
},
|
||||
{
|
||||
icon: Zap,
|
||||
label: t("channels.stats.quotaLeft"),
|
||||
value: status.quota_remaining_today.toLocaleString(),
|
||||
hint: t("channels.stats.quotaLeftHint"),
|
||||
},
|
||||
];
|
||||
|
||||
return (
|
||||
<>
|
||||
<button
|
||||
type="button"
|
||||
ref={pop.refs.setReference}
|
||||
{...pop.getReferenceProps()}
|
||||
className="glass-card glass-hover inline-flex items-center gap-1.5 rounded-lg px-2.5 py-1.5 text-sm text-muted transition"
|
||||
>
|
||||
<BarChart3 className="h-4 w-4" aria-hidden="true" />
|
||||
{t("channels.stats.label")}
|
||||
</button>
|
||||
<PopoverSurface
|
||||
popover={pop}
|
||||
ariaLabel={t("channels.stats.label")}
|
||||
className="glass-menu w-60 rounded-xl p-2 text-xs animate-[popIn_0.16s_ease]"
|
||||
>
|
||||
{rows.map((r) => (
|
||||
<div
|
||||
key={r.label}
|
||||
title={r.hint}
|
||||
className="flex cursor-help items-center gap-2.5 px-1.5 py-1.5"
|
||||
>
|
||||
<r.icon className="h-4 w-4 shrink-0 text-muted" aria-hidden="true" />
|
||||
<span className="flex-1 text-muted">{r.label}</span>
|
||||
<span className="tabular-nums font-medium text-fg">{r.value}</span>
|
||||
</div>
|
||||
))}
|
||||
</PopoverSurface>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -22,7 +22,8 @@ interface TagChipProps {
|
||||
}
|
||||
|
||||
function TagChip({ tag, onClick, title }: TagChipProps) {
|
||||
const cls = "chip-solid shrink-0 text-3xs px-1.5 py-0.5 rounded-full max-w-36 truncate";
|
||||
const cls =
|
||||
"chip-solid shrink-0 text-2xs font-medium px-1.5 py-0.5 rounded-full max-w-36 truncate";
|
||||
return onClick ? (
|
||||
<button
|
||||
type="button"
|
||||
|
||||
@@ -1,25 +1,30 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useEffect, useMemo, useRef, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { qk } from "../lib/queryKeys";
|
||||
import { Ban, History, LayoutGrid, Pencil, RefreshCw, Table, X } from "lucide-react";
|
||||
import { Ban, Eye, EyeOff, History, LayoutGrid, Pencil, RefreshCw, Table, X } from "lucide-react";
|
||||
import ExcludedChannelList from "./ExcludedChannelList";
|
||||
import { api, type ManagedChannel } from "../lib/api";
|
||||
import { notifyYouTubeActionError } from "../lib/youtubeErrors";
|
||||
import { accountKey, LS } from "../lib/storage";
|
||||
import { formatEta } from "../lib/format";
|
||||
import { buildChannelColumns } from "./channelColumns";
|
||||
import { buildChannelColumns, applySyncFilter } from "./channelColumns";
|
||||
import { notify } from "../lib/notifications";
|
||||
import Tooltip from "./Tooltip";
|
||||
import DataTable from "./DataTable";
|
||||
import Tabs, { type TabDef } from "./Tabs";
|
||||
import ChannelLayoutGrid from "./ChannelLayoutGrid";
|
||||
import { LoadingState, StateMessage } from "./QueryState";
|
||||
import { useChannelTagActions } from "../lib/useChannelTagActions";
|
||||
import { useChannelPriority } from "../lib/useChannelPriority";
|
||||
import ViewSwitcher, { type ViewOption } from "./ViewSwitcher";
|
||||
import ColumnSortControl from "./ColumnSortControl";
|
||||
import FilterChipGroup, { filterChipClass } from "./FilterChipGroup";
|
||||
import ActionMenu, { type ActionMenuItem } from "./ActionMenu";
|
||||
import ChannelStatsPopover from "./ChannelStatsPopover";
|
||||
import Pager from "./Pager";
|
||||
import { CHANNEL_LAYOUTS, type ChannelLayout } from "../lib/channelLayout";
|
||||
import { sortRows } from "../lib/columnSort";
|
||||
import { filterRows } from "./tableHeader";
|
||||
import { useCardPager } from "../lib/useCardPager";
|
||||
import { PageToolbar } from "./PageShell";
|
||||
import ChannelDiscovery from "./ChannelDiscovery";
|
||||
@@ -31,15 +36,16 @@ import { useFeedFilters, useFeedFiltersActions } from "./FeedFiltersProvider";
|
||||
import { useWizard } from "./WizardProvider";
|
||||
import { useMe } from "../lib/useMe";
|
||||
|
||||
export type ChannelsView = "subscribed" | "discovery" | "blocked";
|
||||
export type ChannelsView = "subscribed" | "discovery" | "hidden" | "blocked";
|
||||
|
||||
export type ChannelStatusFilter = "all" | "needs_full" | "fully_synced" | "hidden";
|
||||
|
||||
const STATUS_FILTERS: { id: ChannelStatusFilter; labelKey: string }[] = [
|
||||
{ id: "all", labelKey: "channels.filters.all" },
|
||||
{ id: "needs_full", labelKey: "channels.filters.needsFull" },
|
||||
{ id: "fully_synced", labelKey: "channels.filters.fullySynced" },
|
||||
{ id: "hidden", labelKey: "channels.filters.hidden" },
|
||||
// The top sync chips are a controlled view over the subscribed table's `sync` column filter (shared
|
||||
// via ChannelsProvider): each chip writes `filters.sync`, using the SAME vocab + predicate as the
|
||||
// Sync column funnel (channelColumns.tsx) so the two can't drift. "" = all (isActive treats it as
|
||||
// off). fullySynced = recent_synced && backfill_done (matches the cell's own "fully synced" badge).
|
||||
const SYNC_CHIPS: { value: string; labelKey: string }[] = [
|
||||
{ value: "", labelKey: "channels.filters.all" },
|
||||
{ value: "needsFull", labelKey: "channels.filters.needsFull" },
|
||||
{ value: "fully", labelKey: "channels.filters.fullySynced" },
|
||||
];
|
||||
|
||||
const LAYOUT_ICON: Record<ChannelLayout, typeof Table> = {
|
||||
@@ -58,7 +64,7 @@ export default function Channels() {
|
||||
// callers); this page is their primary editor.
|
||||
const {
|
||||
search,
|
||||
statusFilter,
|
||||
filters,
|
||||
view,
|
||||
sort,
|
||||
filtersResetToken,
|
||||
@@ -68,7 +74,7 @@ export default function Channels() {
|
||||
const layout = useChannelLayout();
|
||||
const {
|
||||
setSearch,
|
||||
setStatusFilter,
|
||||
setFilters,
|
||||
setView,
|
||||
setLayout,
|
||||
setSort,
|
||||
@@ -206,41 +212,46 @@ export default function Channels() {
|
||||
notifyYouTubeActionError(e, t("channels.notify.fullHistoryFailed"), onOpenWizard),
|
||||
});
|
||||
|
||||
// Channel-name search (from the shared top SearchBar, via props) + tag-chip filtering, applied
|
||||
// client-side over the status-filtered list.
|
||||
const [tagFilter, setTagFilter] = useState<number[]>([]);
|
||||
// The tag-chip row is a controlled view over the Tags COLUMN filter (`filters.tags`, a string[] of
|
||||
// tag ids), the same way the sync chips drive `filters.sync` — so the top chips and the Tags column
|
||||
// funnel share ONE state and stay in sync (bidirectional). Toggling a chip rewrites `filters.tags`;
|
||||
// filterRows / DataTable then applies it. Channel-name search stays separate (the shared top box).
|
||||
const tagFilter = Array.isArray(filters.tags) ? filters.tags : [];
|
||||
const toggleTagFilter = (tagId: number) => {
|
||||
const id = String(tagId);
|
||||
const next = tagFilter.includes(id) ? tagFilter.filter((x) => x !== id) : [...tagFilter, id];
|
||||
setFilters({ ...filters, tags: next });
|
||||
};
|
||||
// A focus-channel intent (header "without full history" / tag manager) seeds the search box.
|
||||
useEffect(() => {
|
||||
if (focusChannelName) setSearch(focusChannelName);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [focusChannelToken]);
|
||||
// The header's reset intent clears the search + tag chips (skip the initial mount).
|
||||
// The header's reset intent clears the search box (skip the initial mount); the tag chips ride in
|
||||
// `filters.tags`, which goToFullHistory already clears by setting a fresh map.
|
||||
const resetRef = useRef(filtersResetToken);
|
||||
useEffect(() => {
|
||||
if (filtersResetToken === resetRef.current) return;
|
||||
resetRef.current = filtersResetToken;
|
||||
setSearch("");
|
||||
setTagFilter([]);
|
||||
}, [filtersResetToken]);
|
||||
|
||||
// The Sync-status filter stays a top-level chip set (not a column filter) because the
|
||||
// header's "go to full history" deep-link drives it; search/tag filtering + sort + pagination
|
||||
// run over whatever the status chip leaves.
|
||||
const channels = (channelsQuery.data ?? []).filter((c) =>
|
||||
statusFilter === "needs_full"
|
||||
? !c.backfill_done
|
||||
: statusFilter === "fully_synced"
|
||||
? c.backfill_done
|
||||
: statusFilter === "hidden"
|
||||
? c.hidden
|
||||
: true,
|
||||
);
|
||||
// Tag/priority/text filters stay controlled COLUMN filters (`filters.tags` etc.) shared by the top
|
||||
// chips, the column funnels and the cards. The sync-status filter (`filters.sync`) is no longer a
|
||||
// column — R14 retired the Sync column and folded its status into a ✓ charm on Stored — so it's
|
||||
// applied as a pre-filter (`applySyncFilter`) over the rows fed to BOTH the table and the cards,
|
||||
// still driven by the same top chips. Hidden channels have their own tab (excluded here). Only the
|
||||
// name search filters at this step; the remaining column funnels run in DataTable / filterRows below.
|
||||
const hiddenChannels = (channelsQuery.data ?? []).filter((c) => c.hidden);
|
||||
const channels = (channelsQuery.data ?? []).filter((c) => !c.hidden);
|
||||
const q = search.trim().toLowerCase();
|
||||
const visibleChannels = channels.filter((c) => {
|
||||
if (q && !`${c.title ?? ""} ${c.handle ?? ""}`.toLowerCase().includes(q)) return false;
|
||||
if (tagFilter.length && !tagFilter.some((id) => c.tag_ids.includes(id))) return false;
|
||||
return true;
|
||||
});
|
||||
const visibleChannels = channels.filter(
|
||||
(c) => !q || `${c.title ?? ""} ${c.handle ?? ""}`.toLowerCase().includes(q),
|
||||
);
|
||||
// Sync-status pre-filter (top chips → `filters.sync`), applied before the table/cards see the rows.
|
||||
// (Inline, not memoised: `visibleChannels` is a fresh array each render, so a useMemo keyed on it
|
||||
// could never hit its cache — and applySyncFilter is a cheap single pass anyway.)
|
||||
const syncedChannels = applySyncFilter(visibleChannels, filters.sync);
|
||||
const s = statusQuery.data;
|
||||
|
||||
const onView = (c: ManagedChannel) =>
|
||||
@@ -288,88 +299,43 @@ export default function Channels() {
|
||||
|
||||
// Status chips sit in the table's controls row (next to paging) — compact and close to
|
||||
// where they act.
|
||||
const syncFilter = typeof filters.sync === "string" ? filters.sync : "";
|
||||
const statusChips = (
|
||||
<div className="flex flex-wrap items-center gap-1.5">
|
||||
{STATUS_FILTERS.map((f) => (
|
||||
<button
|
||||
key={f.id}
|
||||
onClick={() => setStatusFilter(f.id)}
|
||||
aria-pressed={statusFilter === f.id}
|
||||
className="pill text-xs px-2.5 py-1 rounded-full border"
|
||||
>
|
||||
{t(f.labelKey)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<FilterChipGroup
|
||||
ariaLabel={t("channels.filters.syncLabel")}
|
||||
className="flex flex-wrap items-center gap-1.5"
|
||||
options={SYNC_CHIPS.map((c) => ({ value: c.value, label: t(c.labelKey) }))}
|
||||
value={syncFilter}
|
||||
onChange={(sync) => setFilters({ ...filters, sync })}
|
||||
/>
|
||||
);
|
||||
|
||||
// The catalog-wide stats ride on the right of the tab row (subscribed only), leaving room for
|
||||
// future tabs on the left.
|
||||
const statsBlock = (
|
||||
<div className="flex flex-wrap items-center justify-end gap-x-5 gap-y-1 text-sm text-muted">
|
||||
{s && (
|
||||
<>
|
||||
<Stat
|
||||
label={t("channels.stats.channels")}
|
||||
value={s.channels_total}
|
||||
hint={t("channels.stats.channelsHint")}
|
||||
/>
|
||||
<Stat
|
||||
label={t("channels.stats.recentSynced")}
|
||||
value={`${s.channels_recent_synced}/${s.channels_total}`}
|
||||
hint={t("channels.stats.recentSyncedHint")}
|
||||
/>
|
||||
<Stat
|
||||
label={t("channels.stats.fullHistory")}
|
||||
value={`${s.channels_deep_done}/${s.channels_deep_requested}`}
|
||||
hint={t("channels.stats.fullHistoryHint")}
|
||||
/>
|
||||
{s.deep_pending_count > 0 && (
|
||||
<Stat
|
||||
label={t("channels.stats.left")}
|
||||
value={formatEta(s.deep_eta_seconds)}
|
||||
hint={t("channels.stats.leftHint", { count: s.deep_pending_count })}
|
||||
/>
|
||||
)}
|
||||
<Stat
|
||||
label={t("channels.stats.myVideos")}
|
||||
value={s.my_videos.toLocaleString()}
|
||||
hint={t("channels.stats.myVideosHint")}
|
||||
/>
|
||||
<Stat
|
||||
label={t("channels.stats.quotaLeft")}
|
||||
value={s.quota_remaining_today.toLocaleString()}
|
||||
hint={t("channels.stats.quotaLeftHint")}
|
||||
/>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
// The catalog-wide stats collapse into a compact "Stats" hover chip (right of the tab row, subscribed
|
||||
// only) instead of a crowded inline row.
|
||||
const statsBlock = s ? <ChannelStatsPopover status={s} /> : null;
|
||||
|
||||
// Sync + backfill — now share the tags row (right side).
|
||||
// Catalog-wide, occasional actions — tucked into a shared ⋯ overflow menu so they stop crowding the
|
||||
// tags row. Pending mutations keep the item visible but disabled, with a spinning icon.
|
||||
const catalogActions: ActionMenuItem[] = [
|
||||
{
|
||||
id: "sync",
|
||||
label: t("channels.syncSubscriptions"),
|
||||
title: t("channels.syncSubscriptionsHint"),
|
||||
icon: <RefreshCw className={`h-4 w-4 ${syncSubs.isPending ? "animate-spin" : ""}`} />,
|
||||
onSelect: () => syncSubs.mutate(),
|
||||
disabled: syncSubs.isPending,
|
||||
},
|
||||
{
|
||||
id: "backfill",
|
||||
label: t("channels.backfillEverything"),
|
||||
title: t("channels.backfillEverythingHint"),
|
||||
icon: <History className={`h-4 w-4 ${deepAll.isPending ? "animate-pulse" : ""}`} />,
|
||||
onSelect: () => deepAll.mutate(),
|
||||
disabled: deepAll.isPending,
|
||||
},
|
||||
];
|
||||
const actionButtons = (
|
||||
<div className="flex items-center gap-2 shrink-0">
|
||||
<Tooltip side="bottom" hint={t("channels.syncSubscriptionsHint")}>
|
||||
<button
|
||||
onClick={() => syncSubs.mutate()}
|
||||
disabled={syncSubs.isPending}
|
||||
className="glass-card glass-hover flex items-center gap-2 px-3 py-2 rounded-xl text-sm disabled:opacity-50 transition"
|
||||
>
|
||||
<RefreshCw className={`w-4 h-4 ${syncSubs.isPending ? "animate-spin" : ""}`} />
|
||||
{t("channels.syncSubscriptions")}
|
||||
</button>
|
||||
</Tooltip>
|
||||
<Tooltip side="bottom" hint={t("channels.backfillEverythingHint")}>
|
||||
<button
|
||||
onClick={() => deepAll.mutate()}
|
||||
disabled={deepAll.isPending}
|
||||
className="glass-card glass-hover flex items-center gap-2 px-3 py-2 rounded-xl text-sm disabled:opacity-50 transition"
|
||||
>
|
||||
<History className={`w-4 h-4 ${deepAll.isPending ? "animate-pulse" : ""}`} />
|
||||
{t("channels.backfillEverything")}
|
||||
</button>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<ActionMenu ariaLabel={t("channels.actionsLabel")} items={catalogActions} />
|
||||
);
|
||||
|
||||
// Table / cards density — shared by the subscribed + discovery tables (blocked is a chip list, no
|
||||
@@ -407,7 +373,11 @@ export default function Channels() {
|
||||
// Only computed for cards: in table mode DataTable sorts and slices internally, so doing it here
|
||||
// too would copy+sort the whole ~300-row array on every render for a result nothing reads.
|
||||
const isCards = layout === "cards";
|
||||
const sortedChannels = isCards ? sortRows(visibleChannels, columns, sort) : visibleChannels;
|
||||
// Cards have no column headers, so apply the shared column filters (sync + any funnel set in table
|
||||
// mode) here via the exported predicate — the same filterRows DataTable runs internally — then sort.
|
||||
const sortedChannels = isCards
|
||||
? sortRows(filterRows(syncedChannels, columns, filters), columns, sort)
|
||||
: syncedChannels;
|
||||
const {
|
||||
paged: pagedCards,
|
||||
pagerProps,
|
||||
@@ -420,23 +390,25 @@ export default function Channels() {
|
||||
// handled by the hook's clamp instead.
|
||||
useEffect(() => {
|
||||
resetPage();
|
||||
}, [statusFilter, search, tagFilter, sort, resetPage]);
|
||||
}, [filters, search, sort, resetPage]);
|
||||
const cardPager = <Pager {...pagerProps} />;
|
||||
|
||||
const viewTabs: TabDef[] = useMemo(
|
||||
() =>
|
||||
(["subscribed", "discovery", "hidden", "blocked"] as ChannelsView[]).map((v) => ({
|
||||
id: v,
|
||||
label: t(`channels.tabs.${v}`),
|
||||
})),
|
||||
[t],
|
||||
);
|
||||
const tabs = (
|
||||
<div className="px-4 pt-4 max-w-384 mx-auto flex items-center justify-between gap-4 flex-wrap">
|
||||
<div className="flex flex-wrap items-center gap-1.5">
|
||||
{(["subscribed", "discovery", "blocked"] as ChannelsView[]).map((v) => (
|
||||
<button
|
||||
key={v}
|
||||
onClick={() => setView(v)}
|
||||
aria-pressed={view === v}
|
||||
className="pill text-sm px-3 py-1.5 rounded-full border"
|
||||
>
|
||||
{t(`channels.tabs.${v}`)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="px-4 pt-2 max-w-384 mx-auto flex items-center justify-between gap-4 flex-wrap">
|
||||
<Tabs
|
||||
tabs={viewTabs}
|
||||
active={view}
|
||||
onChange={(id) => setView(id as ChannelsView)}
|
||||
className="flex flex-wrap items-center gap-1"
|
||||
/>
|
||||
<div className="flex items-center gap-3">
|
||||
{view === "subscribed" && statsBlock}
|
||||
{view === "subscribed" && layout === "cards" && sortControl}
|
||||
@@ -458,41 +430,35 @@ export default function Channels() {
|
||||
search lives in the shared top SearchBar, tag create/delete in the manager. */}
|
||||
<div className="flex items-center justify-between gap-4 flex-wrap">
|
||||
<div className="flex flex-wrap items-center gap-1.5">
|
||||
{/* The "TAGS" label IS the tags-editor trigger (pencil), replacing the separate
|
||||
"Manage tags" button; the tooltip explains what tags are + that it's clickable. */}
|
||||
<Tooltip hint={t("channels.tags.yourTagsHint")}>
|
||||
<span className="text-xs uppercase tracking-wide text-muted mr-1 cursor-help underline decoration-dotted decoration-muted/40 underline-offset-4">
|
||||
{t("channels.tags.yourTags")}
|
||||
</span>
|
||||
<button
|
||||
onClick={() => setTagManagerOpen(true)}
|
||||
className="mr-1 inline-flex items-center gap-1.5 rounded-lg border border-border px-2 py-1 text-xs uppercase tracking-wide text-muted transition hover:border-accent hover:text-accent"
|
||||
>
|
||||
<Pencil className="h-3 w-3" />
|
||||
{t("channels.tags.label")}
|
||||
</button>
|
||||
</Tooltip>
|
||||
{userTags.map((tg) => {
|
||||
const active = tagFilter.includes(tg.id);
|
||||
return (
|
||||
<button
|
||||
key={tg.id}
|
||||
onClick={() =>
|
||||
setTagFilter((f) => (active ? f.filter((x) => x !== tg.id) : [...f, tg.id]))
|
||||
}
|
||||
aria-pressed={active}
|
||||
className="pill text-xs px-2 py-1 rounded-full border"
|
||||
>
|
||||
{tg.name}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
{userTags.map((tg) => (
|
||||
<button
|
||||
key={tg.id}
|
||||
onClick={() => toggleTagFilter(tg.id)}
|
||||
aria-pressed={tagFilter.includes(String(tg.id))}
|
||||
className={filterChipClass}
|
||||
>
|
||||
{tg.name}
|
||||
</button>
|
||||
))}
|
||||
{tagFilter.length > 0 && (
|
||||
<button
|
||||
onClick={() => setTagFilter([])}
|
||||
onClick={() => setFilters({ ...filters, tags: [] })}
|
||||
className="text-xs px-2 py-1 text-muted hover:text-accent transition"
|
||||
>
|
||||
{t("datatable.clear")}
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
onClick={() => setTagManagerOpen(true)}
|
||||
className="inline-flex items-center gap-1 text-xs px-2 py-1 rounded-full border border-dashed border-border text-muted hover:text-accent hover:border-accent transition"
|
||||
>
|
||||
<Pencil className="w-3 h-3" />
|
||||
{t("channels.tags.manage")}
|
||||
</button>
|
||||
</div>
|
||||
{actionButtons}
|
||||
</div>
|
||||
@@ -521,37 +487,33 @@ export default function Channels() {
|
||||
) : view === "blocked" ? (
|
||||
/* Blocked channels — videos from these are hidden everywhere and dropped from live search. */
|
||||
<div className="px-4 pb-4 pt-3 max-w-384 mx-auto">
|
||||
{(blockedQuery.data?.length ?? 0) === 0 ? (
|
||||
<p className="text-sm text-muted py-8">{t("channels.blocked.empty")}</p>
|
||||
) : (
|
||||
<>
|
||||
<div className="flex items-center gap-2 mb-3">
|
||||
<Ban className="w-4 h-4 text-muted" />
|
||||
<span className="text-xs uppercase tracking-wide text-muted">
|
||||
{t("channels.blocked.title")}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{blockedQuery.data!.map((c) => (
|
||||
<span
|
||||
key={c.id}
|
||||
className="inline-flex items-center gap-1.5 text-xs pl-2.5 pr-1 py-1 rounded-full bg-card border border-border"
|
||||
>
|
||||
{c.title ?? c.id}
|
||||
<button
|
||||
onClick={() => unblock.mutate(c.id)}
|
||||
title={t("channel.unblock")}
|
||||
aria-label={t("channel.unblock")}
|
||||
className="rounded-full p-0.5 text-muted hover:text-danger transition"
|
||||
>
|
||||
<X className="w-3 h-3" />
|
||||
</button>
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
<p className="text-2xs text-muted mt-2">{t("channels.blocked.hint")}</p>
|
||||
</>
|
||||
)}
|
||||
<ExcludedChannelList
|
||||
channels={blockedQuery.data ?? []}
|
||||
icon={<Ban className="w-4 h-4 text-muted" />}
|
||||
title={t("channels.blocked.title")}
|
||||
hint={t("channels.blocked.hint")}
|
||||
emptyText={t("channels.blocked.empty")}
|
||||
actionLabel={t("channel.unblock")}
|
||||
actionIcon={<X className="w-3.5 h-3.5" />}
|
||||
onAction={(id) => unblock.mutate(id)}
|
||||
onView={(c) => onViewChannel(c.id, c.title ?? t("channels.row.thisChannel"))}
|
||||
/>
|
||||
</div>
|
||||
) : view === "hidden" ? (
|
||||
/* Hidden channels — kept out of the Mine feed but still in the shared Library; managed here
|
||||
instead of dimmed in the subscribed table. Same card grid as Blocked (ExcludedChannelList). */
|
||||
<div className="px-4 pb-4 pt-3 max-w-384 mx-auto">
|
||||
<ExcludedChannelList
|
||||
channels={hiddenChannels}
|
||||
icon={<EyeOff className="w-4 h-4 text-muted" />}
|
||||
title={t("channels.hidden.title")}
|
||||
hint={t("channels.hidden.hint")}
|
||||
emptyText={t("channels.hidden.empty")}
|
||||
actionLabel={t("channels.hidden.unhide")}
|
||||
actionIcon={<Eye className="w-3.5 h-3.5" />}
|
||||
onAction={(id) => patch.mutate({ id, body: { hidden: false } })}
|
||||
onView={(c) => onViewChannel(c.id, c.title ?? t("channels.row.thisChannel"))}
|
||||
/>
|
||||
</div>
|
||||
) : (
|
||||
/* No top padding: the sticky header sits flush at the scroller top so it doesn't shift when
|
||||
@@ -570,7 +532,7 @@ export default function Channels() {
|
||||
<LoadingState label={t("channels.loading")} />
|
||||
) : layout === "table" ? (
|
||||
<DataTable
|
||||
rows={visibleChannels}
|
||||
rows={syncedChannels}
|
||||
columns={columns}
|
||||
rowKey={(c) => c.id}
|
||||
persistKey={accountKey(LS.channelsTable) ?? undefined}
|
||||
@@ -580,7 +542,9 @@ export default function Channels() {
|
||||
controlsBandClassName="px-4 max-w-384 mx-auto"
|
||||
sort={sort}
|
||||
onSortChange={setSort}
|
||||
rowClassName={(c) => (c.hidden ? "opacity-60" : "")}
|
||||
filters={filters}
|
||||
onFiltersChange={setFilters}
|
||||
fixedLayout
|
||||
emptyText={t("channels.empty")}
|
||||
/>
|
||||
) : (
|
||||
@@ -591,7 +555,6 @@ export default function Channels() {
|
||||
leadKey="priority"
|
||||
aboutKey="about"
|
||||
actionsKey="actions"
|
||||
rowClassName={(c) => (c.hidden ? "opacity-60" : "")}
|
||||
emptyText={t("channels.empty")}
|
||||
/>
|
||||
)}
|
||||
@@ -600,13 +563,3 @@ export default function Channels() {
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function Stat({ label, value, hint }: { label: string; value: string | number; hint?: string }) {
|
||||
return (
|
||||
<Tooltip hint={hint ?? ""}>
|
||||
<span className={hint ? "cursor-help" : ""}>
|
||||
<span className="text-fg font-semibold">{value}</span> {label}
|
||||
</span>
|
||||
</Tooltip>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// ChannelsProvider's tab-history behaviour: setView pushes a `_chTab` history entry + persists the tab,
|
||||
// and a genuine browser Back (popstate carrying `_chTab`) restores + re-persists it. This wiring was
|
||||
// re-routed through usePersistedTab in R13 S2b, so it's exercised here rather than only by hand E2E.
|
||||
//
|
||||
// Uses REAL storage (jsdom localStorage/sessionStorage) with the tab's account pinned to 1, so
|
||||
// usePersistedTab / useSortState / useFilterState all resolve their account-scoped keys; only
|
||||
// NavigationProvider (the sole runtime dependency) is mocked, pinned to the channels page.
|
||||
import { describe, it, expect, beforeEach, vi } from "vitest";
|
||||
import { renderHook, act } from "@testing-library/react";
|
||||
import type { ReactNode } from "react";
|
||||
|
||||
vi.mock("./NavigationProvider", () => ({
|
||||
useNavigation: () => ({ page: "channels" }),
|
||||
useNavigationActions: () => ({ setPage: vi.fn() }),
|
||||
}));
|
||||
|
||||
import { ChannelsProvider, useChannels, useChannelsActions } from "./ChannelsProvider";
|
||||
|
||||
const wrapper = ({ children }: { children: ReactNode }) => (
|
||||
<ChannelsProvider>{children}</ChannelsProvider>
|
||||
);
|
||||
const render = () => renderHook(() => ({ ...useChannels(), ...useChannelsActions() }), { wrapper });
|
||||
const VIEW_KEY = "siftlode.channelsView.1"; // account-scoped (accountKey suffixes the pinned id)
|
||||
|
||||
beforeEach(() => {
|
||||
localStorage.clear();
|
||||
sessionStorage.setItem("siftlode.activeAccount", "1"); // pin the tab's account
|
||||
window.history.replaceState({ sfPage: "channels" }, ""); // setView only owns an entry on this page
|
||||
});
|
||||
|
||||
describe("ChannelsProvider tab history", () => {
|
||||
it("setView pushes a _chTab history entry and persists the tab", () => {
|
||||
const { result } = render();
|
||||
expect(result.current.view).toBe("subscribed"); // default
|
||||
|
||||
act(() => result.current.setView("hidden"));
|
||||
|
||||
expect(result.current.view).toBe("hidden");
|
||||
expect(window.history.state).toMatchObject({ sfPage: "channels", _chTab: "hidden" });
|
||||
expect(localStorage.getItem(VIEW_KEY)).toBe("hidden");
|
||||
});
|
||||
|
||||
it("does NOT push a new entry when re-selecting the current tab", () => {
|
||||
const { result } = render();
|
||||
const spy = vi.spyOn(window.history, "pushState");
|
||||
act(() => result.current.setView("subscribed")); // already the current tab
|
||||
expect(spy).not.toHaveBeenCalled();
|
||||
spy.mockRestore();
|
||||
});
|
||||
|
||||
it("a genuine Back (popstate carrying _chTab) restores and re-persists that tab", () => {
|
||||
const { result } = render();
|
||||
act(() => result.current.setView("blocked"));
|
||||
expect(result.current.view).toBe("blocked");
|
||||
|
||||
// Browser Back to an entry stamped with a different tab.
|
||||
act(() =>
|
||||
window.dispatchEvent(new PopStateEvent("popstate", { state: { _chTab: "discovery" } })),
|
||||
);
|
||||
|
||||
expect(result.current.view).toBe("discovery");
|
||||
expect(localStorage.getItem(VIEW_KEY)).toBe("discovery");
|
||||
});
|
||||
|
||||
it("ignores a popstate for an entry outside the manager (no _chTab)", () => {
|
||||
const { result } = render();
|
||||
act(() => result.current.setView("hidden"));
|
||||
act(() => window.dispatchEvent(new PopStateEvent("popstate", { state: { sfPage: "feed" } })));
|
||||
expect(result.current.view).toBe("hidden"); // unchanged
|
||||
});
|
||||
});
|
||||
@@ -8,41 +8,47 @@ import {
|
||||
useState,
|
||||
type ReactNode,
|
||||
} from "react";
|
||||
import { getAccountRaw, LS, readAccount, setAccountRaw, writeAccount } from "../lib/storage";
|
||||
import { getAccountRaw, LS } from "../lib/storage";
|
||||
import { parseChannelLayout, type ChannelLayout } from "../lib/channelLayout";
|
||||
import { parseSortState, type SortState } from "../lib/columnSort";
|
||||
import { type SortState } from "../lib/columnSort";
|
||||
import { useSortState } from "../lib/useSortState";
|
||||
import { useFilterState } from "../lib/useFilterState";
|
||||
import { usePersistedTab } from "./Tabs";
|
||||
import type { FilterMap } from "./tableHeader";
|
||||
import { useNavigation, useNavigationActions } from "./NavigationProvider";
|
||||
import type { ChannelStatusFilter, ChannelsView } from "./Channels";
|
||||
import type { ChannelsView } from "./Channels";
|
||||
|
||||
// Owns the Channels module's shared local state — the rail-less manager's name filter, status chip,
|
||||
// active tab, the "drop a stale column filter" token, and the focus-a-channel intent — lifted out of
|
||||
// Owns the Channels module's shared local state — the rail-less manager's name filter, column-filter
|
||||
// map, active tab, the "drop a stale column filter" token, and the focus-a-channel intent — lifted out of
|
||||
// App so the Channels page, the header search box, and the cross-module callers (the feed filter
|
||||
// sidebar and the notifications inbox both "focus this channel in the manager") read it via hooks
|
||||
// instead of prop-drilling through App. Mirrors FeedFiltersProvider (split State + Actions contexts,
|
||||
// stable useCallback setters, one-line hooks; composed in main.tsx around <App/>).
|
||||
//
|
||||
// Nested INSIDE NavigationProvider so its actions can drive navigation: focusChannel and
|
||||
// goToFullHistory both jump to the Channels page (and force the subscriptions tab). status / view are
|
||||
// goToFullHistory both jump to the Channels page (and force the subscriptions tab). filters / view are
|
||||
// per-account persisted (localStorage); search / focus / the reset token are ephemeral. An account
|
||||
// switch does a full reload, so init-from-storage is all the hydration the persisted ones need.
|
||||
|
||||
const STATUS_FILTERS: readonly ChannelStatusFilter[] = [
|
||||
"needs_full",
|
||||
"fully_synced",
|
||||
"hidden",
|
||||
"all",
|
||||
];
|
||||
|
||||
function clampStatus(v: string | null): ChannelStatusFilter {
|
||||
return STATUS_FILTERS.includes(v as ChannelStatusFilter) ? (v as ChannelStatusFilter) : "all";
|
||||
}
|
||||
function clampView(v: string | null): ChannelsView {
|
||||
return v === "discovery" ? "discovery" : v === "blocked" ? "blocked" : "subscribed";
|
||||
return v === "discovery" || v === "blocked" || v === "hidden" ? v : "subscribed";
|
||||
}
|
||||
|
||||
// One-time carry-forward of the retired single-value sync chip (LS.legacyChannelFilter, vocab
|
||||
// all/needs_full/fully_synced) into the new filter map's `sync` key (vocab ""/needsFull/fully), so a
|
||||
// user who had a chip pinned keeps it. Only consulted when channelsFilterMap is still empty.
|
||||
function migrateSyncChip(): FilterMap | null {
|
||||
const old = getAccountRaw(LS.legacyChannelFilter);
|
||||
if (old === "needs_full") return { sync: "needsFull" };
|
||||
if (old === "fully_synced") return { sync: "fully" };
|
||||
return null;
|
||||
}
|
||||
|
||||
type ChannelsState = {
|
||||
search: string;
|
||||
statusFilter: ChannelStatusFilter;
|
||||
// The subscribed table's column filters, lifted out of DataTable so the top status chips, the Sync
|
||||
// column funnel, and the card layout share ONE filter state (the sync chip lives at `filters.sync`).
|
||||
filters: FilterMap;
|
||||
view: ChannelsView;
|
||||
// The subscribed table's sort, lifted out of DataTable so the card layout shares it.
|
||||
sort: SortState;
|
||||
@@ -57,7 +63,7 @@ type ChannelsState = {
|
||||
|
||||
interface ChannelsActions {
|
||||
setSearch: (q: string) => void;
|
||||
setStatusFilter: (f: ChannelStatusFilter) => void;
|
||||
setFilters: (f: FilterMap) => void;
|
||||
setView: (v: ChannelsView) => void;
|
||||
setLayout: (l: ChannelLayout) => void;
|
||||
setSort: (s: SortState) => void;
|
||||
@@ -70,7 +76,7 @@ interface ChannelsActions {
|
||||
|
||||
const StateContext = createContext<ChannelsState>({
|
||||
search: "",
|
||||
statusFilter: "all",
|
||||
filters: {},
|
||||
view: "subscribed",
|
||||
sort: null,
|
||||
filtersResetToken: 0,
|
||||
@@ -83,7 +89,7 @@ const StateContext = createContext<ChannelsState>({
|
||||
const LayoutContext = createContext<ChannelLayout>("table");
|
||||
const ActionsContext = createContext<ChannelsActions>({
|
||||
setSearch: () => {},
|
||||
setStatusFilter: () => {},
|
||||
setFilters: () => {},
|
||||
setView: () => {},
|
||||
setLayout: () => {},
|
||||
setSort: () => {},
|
||||
@@ -108,18 +114,20 @@ export function ChannelsProvider({ children }: { children: ReactNode }) {
|
||||
const { setPage } = useNavigationActions();
|
||||
|
||||
const [search, setSearchState] = useState("");
|
||||
const [statusFilter, setStatusFilterState] = useState<ChannelStatusFilter>(() =>
|
||||
clampStatus(getAccountRaw(LS.channelFilter)),
|
||||
);
|
||||
const [view, setViewState] = useState<ChannelsView>(() =>
|
||||
clampView(getAccountRaw(LS.channelsView)),
|
||||
);
|
||||
const [layout, setLayoutState] = useState<ChannelLayout>(() =>
|
||||
parseChannelLayout(getAccountRaw(LS.channelLayout)),
|
||||
);
|
||||
const [sort, setSortState] = useState<SortState>(() =>
|
||||
parseSortState(readAccount(LS.channelSort, null)),
|
||||
);
|
||||
// The subscribed table's column-filter map, owned here (lifted out of DataTable) so the top status
|
||||
// chips, the Sync column funnel, and the card layout share one state; useFilterState persists it.
|
||||
const [filters, setFilters] = useFilterState(LS.channelsFilterMap, migrateSyncChip);
|
||||
// view + layout are persisted tabs (usePersistedTab = the shared account-scoped tab-state hook), so
|
||||
// they get init-from-storage AND the account-pin re-hydrate for free (this provider mounts above
|
||||
// <App>, which pins the account only after /api/me). Stored un-clamped; clampView/parseChannelLayout
|
||||
// heal them for display. setView layers the history logic (below) on top of the raw setter.
|
||||
const [viewRaw, setViewRaw] = usePersistedTab(LS.channelsView, "subscribed");
|
||||
const view = clampView(viewRaw);
|
||||
const [layoutRaw, setLayout] = usePersistedTab(LS.channelLayout, "table");
|
||||
const layout = parseChannelLayout(layoutRaw);
|
||||
// Sort is owned here (lifted out of DataTable) so the card layout shares it; ColumnSortControl heals
|
||||
// it against the live columns for display, so the provider stores it un-healed (no options passed).
|
||||
const [sort, setSort] = useSortState(LS.channelSort);
|
||||
const [filtersResetToken, setFiltersResetToken] = useState(0);
|
||||
const [focusName, setFocusName] = useState<string | null>(null);
|
||||
const [focusToken, setFocusToken] = useState(0);
|
||||
@@ -131,37 +139,26 @@ export function ChannelsProvider({ children }: { children: ReactNode }) {
|
||||
viewRef.current = view;
|
||||
|
||||
const setSearch = useCallback((q: string) => setSearchState(q), []);
|
||||
const setStatusFilter = useCallback((f: ChannelStatusFilter) => {
|
||||
setStatusFilterState(f);
|
||||
setAccountRaw(LS.channelFilter, f);
|
||||
}, []);
|
||||
// The active tab rides in history.state (`_chTab`), the same way the channel page (`_chan`) and the
|
||||
// live YouTube search (`_yt`) do — so Back steps back through the tabs you actually visited instead
|
||||
// of jumping straight out to the previous module.
|
||||
const setView = useCallback((v: ChannelsView) => {
|
||||
// Re-selecting the tab you are already on is not a navigation — bail before pushing, or clicking
|
||||
// the active tab (and goToFullHistory forcing "subscribed") stacks Back entries that do nothing.
|
||||
if (v === viewRef.current) return;
|
||||
setViewState(v);
|
||||
setAccountRaw(LS.channelsView, v);
|
||||
// Only own an entry while the manager is already on screen. Arriving from elsewhere
|
||||
// (focusChannel / goToFullHistory) lets setPage push the entry — the stamp effect labels it — so
|
||||
// one navigation never costs two Back presses.
|
||||
if (window.history.state?.sfPage === "channels") {
|
||||
// A FRESH state, like setPage's: carrying the old entry's `_chan`/`_yt` markers over would make
|
||||
// Back re-open the channel overlay or a spent search on top of the tab.
|
||||
window.history.pushState({ sfPage: "channels", _chTab: v }, "");
|
||||
}
|
||||
}, []);
|
||||
const setLayout = useCallback((l: ChannelLayout) => {
|
||||
setLayoutState(l);
|
||||
setAccountRaw(LS.channelLayout, l);
|
||||
}, []);
|
||||
const setSort = useCallback((s: SortState) => {
|
||||
setSortState(s);
|
||||
writeAccount(LS.channelSort, s);
|
||||
}, []);
|
||||
|
||||
const setView = useCallback(
|
||||
(v: ChannelsView) => {
|
||||
// Re-selecting the tab you are already on is not a navigation — bail before pushing, or clicking
|
||||
// the active tab (and goToFullHistory forcing "subscribed") stacks Back entries that do nothing.
|
||||
if (v === viewRef.current) return;
|
||||
setViewRaw(v); // persists + guards the account-pin re-hydrate (via usePersistedTab)
|
||||
// Only own an entry while the manager is already on screen. Arriving from elsewhere
|
||||
// (focusChannel / goToFullHistory) lets setPage push the entry — the stamp effect labels it — so
|
||||
// one navigation never costs two Back presses.
|
||||
if (window.history.state?.sfPage === "channels") {
|
||||
// A FRESH state, like setPage's: carrying the old entry's `_chan`/`_yt` markers over would make
|
||||
// Back re-open the channel overlay or a spent search on top of the tab.
|
||||
window.history.pushState({ sfPage: "channels", _chTab: v }, "");
|
||||
}
|
||||
},
|
||||
[setViewRaw],
|
||||
);
|
||||
const focusChannel = useCallback(
|
||||
(name: string) => {
|
||||
setFocusName(name);
|
||||
@@ -172,11 +169,13 @@ export function ChannelsProvider({ children }: { children: ReactNode }) {
|
||||
[setView, setPage],
|
||||
);
|
||||
const goToFullHistory = useCallback(() => {
|
||||
setStatusFilter("needs_full");
|
||||
setView("subscribed"); // the status filter applies to the subscriptions tab
|
||||
setFiltersResetToken((n) => n + 1); // drop any stale column filter hiding the rows
|
||||
// One controlled map replaces the old status chip: sync=needsFull, and dropping every other
|
||||
// column filter (the whole map is set fresh) so no stale funnel hides the rows we're sent to see.
|
||||
setFilters({ sync: "needsFull" });
|
||||
setView("subscribed"); // the sync filter applies to the subscriptions tab
|
||||
setFiltersResetToken((n) => n + 1); // also clear the ephemeral name search + tag chips
|
||||
setPage("channels");
|
||||
}, [setStatusFilter, setView, setPage]);
|
||||
}, [setFilters, setView, setPage]);
|
||||
|
||||
// Clear the manager focus when leaving the Channels page so it doesn't re-apply on a later visit.
|
||||
useEffect(() => {
|
||||
@@ -196,29 +195,27 @@ export function ChannelsProvider({ children }: { children: ReactNode }) {
|
||||
function onPop(e: PopStateEvent) {
|
||||
const tab = e.state?._chTab;
|
||||
if (!tab) return; // an entry outside the manager — leave the tab alone
|
||||
const next = clampView(tab as string);
|
||||
setViewState(next);
|
||||
setAccountRaw(LS.channelsView, next);
|
||||
setViewRaw(clampView(tab as string)); // persists the restored tab (setViewRaw is stable)
|
||||
}
|
||||
window.addEventListener("popstate", onPop);
|
||||
return () => window.removeEventListener("popstate", onPop);
|
||||
}, []);
|
||||
}, [setViewRaw]);
|
||||
|
||||
const state = useMemo<ChannelsState>(
|
||||
() => ({ search, statusFilter, view, sort, filtersResetToken, focusName, focusToken }),
|
||||
[search, statusFilter, view, sort, filtersResetToken, focusName, focusToken],
|
||||
() => ({ search, filters, view, sort, filtersResetToken, focusName, focusToken }),
|
||||
[search, filters, view, sort, filtersResetToken, focusName, focusToken],
|
||||
);
|
||||
const actions = useMemo<ChannelsActions>(
|
||||
() => ({
|
||||
setSearch,
|
||||
setStatusFilter,
|
||||
setFilters,
|
||||
setView,
|
||||
setLayout,
|
||||
setSort,
|
||||
focusChannel,
|
||||
goToFullHistory,
|
||||
}),
|
||||
[setSearch, setStatusFilter, setView, setLayout, setSort, focusChannel, goToFullHistory],
|
||||
[setSearch, setFilters, setView, setLayout, setSort, focusChannel, goToFullHistory],
|
||||
);
|
||||
|
||||
return (
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { ArrowDown, ArrowUp } from "lucide-react";
|
||||
import { type Column } from "./DataTable";
|
||||
import { type SortState } from "../lib/columnSort";
|
||||
import { optionsFromColumns } from "../lib/sortOptions";
|
||||
import SortControl from "./SortControl";
|
||||
|
||||
// A sort control for views without clickable column headers (the channel manager's card layout):
|
||||
// a dropdown of the sortable columns + an asc/desc toggle, mirroring the feed toolbar's sort. It's
|
||||
// driven by the SAME `SortState` the DataTable header uses, so table and cards stay in sync. The
|
||||
// dropdown's options come straight from the columns' `sortable`/`sortValue` metadata — no new list.
|
||||
// The headerless sort control for views without clickable column headers (the channel manager's and
|
||||
// discovery's card layouts). Now a thin adapter over the generic `SortControl`, feeding it the
|
||||
// column-derived options and a "no sort" default — kept as its own component so the existing call
|
||||
// sites stay untouched and table/cards keep sharing one `SortState`.
|
||||
export default function ColumnSortControl<T>({
|
||||
columns,
|
||||
sort,
|
||||
@@ -25,44 +26,15 @@ export default function ColumnSortControl<T>({
|
||||
ascLabel: string;
|
||||
descLabel: string;
|
||||
}) {
|
||||
const sortable = columns.filter((c) => c.sortable && c.sortValue);
|
||||
// A stored sort can name a column this table no longer offers (renamed, or `sortable` dropped).
|
||||
// `sortRows` already leaves the rows unsorted then, so treat it as no sort here too — otherwise the
|
||||
// select falls back to showing "default" while a direction arrow claims a sort that isn't applied.
|
||||
const active = sortable.some((c) => c.key === sort?.key) ? sort : null;
|
||||
return (
|
||||
<div className="flex items-center gap-1">
|
||||
<select
|
||||
value={active?.key ?? ""}
|
||||
onChange={(e) => {
|
||||
const key = e.target.value;
|
||||
// Keep the direction when re-picking the same column; a new column starts ascending.
|
||||
onChange(key ? { key, dir: active?.key === key ? active.dir : "asc" } : null);
|
||||
}}
|
||||
aria-label={label}
|
||||
className="bg-surface border border-border rounded-lg px-2 py-1.5 text-sm focus:outline-hidden focus:ring-2 focus:ring-accent/40"
|
||||
>
|
||||
<option value="">{defaultLabel}</option>
|
||||
{sortable.map((c) => (
|
||||
<option key={c.key} value={c.key}>
|
||||
{c.header}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
{active && (
|
||||
<button
|
||||
onClick={() => onChange({ key: active.key, dir: active.dir === "asc" ? "desc" : "asc" })}
|
||||
title={active.dir === "asc" ? ascLabel : descLabel}
|
||||
aria-label={active.dir === "asc" ? ascLabel : descLabel}
|
||||
className="shrink-0 p-1.5 rounded-lg border border-border bg-card text-fg hover:border-accent hover:text-accent active:translate-y-px transition"
|
||||
>
|
||||
{active.dir === "asc" ? (
|
||||
<ArrowUp className="w-4 h-4" />
|
||||
) : (
|
||||
<ArrowDown className="w-4 h-4" />
|
||||
)}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<SortControl
|
||||
options={optionsFromColumns(columns)}
|
||||
sort={sort}
|
||||
onChange={onChange}
|
||||
label={label}
|
||||
defaultLabel={defaultLabel}
|
||||
ascLabel={ascLabel}
|
||||
descLabel={descLabel}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
// A small count pill shown beside a tab label or filter chip (pending requests, item counts). One
|
||||
// source for the count-badge token so Tabs and FilterChipGroup can't drift. The `active` variant sits
|
||||
// on an accent-filled parent (a pressed tab/chip), so it tints against accent-fg; the resting variant
|
||||
// carries its own accent tint. Callers guard on `count > 0` — this just renders the pill.
|
||||
export default function CountBadge({ count, active }: { count: number; active?: boolean }) {
|
||||
return (
|
||||
<span
|
||||
className={`tabular-nums text-2xs leading-none px-1.5 py-0.5 rounded-full ${
|
||||
active ? "bg-accent-fg/20" : "bg-accent/20 text-accent"
|
||||
}`}
|
||||
>
|
||||
{count}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// The R14 zoom-fit foundation: DataTable's opt-in `fixedLayout` renders a <colgroup> and puts the
|
||||
// table into `table-layout: fixed`, so columns with a width keep it and the rest (fluid text columns)
|
||||
// split the remainder + truncate instead of pushing the table wider than its container. Off by
|
||||
// default so existing content-sized tables are untouched.
|
||||
import { describe, it, expect, afterEach, vi } from "vitest";
|
||||
import { render, cleanup } from "@testing-library/react";
|
||||
import DataTable, { type Column } from "./DataTable";
|
||||
|
||||
vi.mock("react-i18next", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("react-i18next")>();
|
||||
return { ...actual, useTranslation: () => ({ t: (key: string) => key }) };
|
||||
});
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
type Row = { a: string; b: string };
|
||||
const columns: Column<Row>[] = [
|
||||
{ key: "a", header: "A", width: "5rem", render: (r) => r.a },
|
||||
{ key: "b", header: "B", render: (r) => r.b }, // fluid — no width
|
||||
];
|
||||
const rows: Row[] = [{ a: "1", b: "hello" }];
|
||||
|
||||
describe("DataTable fixedLayout", () => {
|
||||
it("renders a width-bearing colgroup and table-fixed only when fixedLayout is set", () => {
|
||||
const { container } = render(
|
||||
<DataTable rows={rows} columns={columns} rowKey={(r) => r.a} fixedLayout />,
|
||||
);
|
||||
const table = container.querySelector("table");
|
||||
expect(table?.className).toContain("table-fixed");
|
||||
|
||||
const cols = container.querySelectorAll("colgroup col");
|
||||
expect(cols).toHaveLength(2);
|
||||
expect((cols[0] as HTMLElement).style.width).toBe("5rem"); // fixed column keeps its width
|
||||
expect((cols[1] as HTMLElement).style.width).toBe(""); // fluid column: no width, splits remainder
|
||||
});
|
||||
|
||||
it("omits the colgroup and stays auto-layout by default", () => {
|
||||
const { container } = render(<DataTable rows={rows} columns={columns} rowKey={(r) => r.a} />);
|
||||
expect(container.querySelector("colgroup")).toBeNull();
|
||||
expect(container.querySelector("table")?.className).not.toContain("table-fixed");
|
||||
});
|
||||
});
|
||||
@@ -25,6 +25,11 @@ interface Persisted {
|
||||
size?: number;
|
||||
}
|
||||
|
||||
// Stable empty-map identity for the controlled-but-value-omitted fallback, so `filters` keeps a
|
||||
// constant reference across renders (a fresh `{}` would churn the persist effect + re-filter every
|
||||
// render). Mirrors controlled sort's stable `null` fallback.
|
||||
const EMPTY_FILTERS: FilterMap = {};
|
||||
|
||||
function loadPersist(key?: string): Persisted {
|
||||
const empty: Persisted = { sort: null, filters: {}, page: 0 };
|
||||
if (!key) return empty;
|
||||
@@ -53,6 +58,9 @@ export default function DataTable<T>({
|
||||
resetFiltersToken,
|
||||
sort: sortProp,
|
||||
onSortChange,
|
||||
filters: filtersProp,
|
||||
onFiltersChange,
|
||||
fixedLayout = false,
|
||||
}: {
|
||||
rows: T[];
|
||||
columns: Column<T>[];
|
||||
@@ -82,6 +90,19 @@ export default function DataTable<T>({
|
||||
// persistKey-backed sort.
|
||||
sort?: SortState;
|
||||
onSortChange?: (next: SortState) => void;
|
||||
// CONTROLLED filters (optional): symmetric to controlled sort. When `onFiltersChange` is given the
|
||||
// caller owns the column-filter map (and its persistence) — so the channel manager's top status
|
||||
// chips, the Sync column funnel, and the card layout all share ONE filter state — and the table
|
||||
// reflects `filters` and reports funnel changes instead of keeping its own. Omit both for the
|
||||
// default internal, persistKey-backed filters.
|
||||
filters?: FilterMap;
|
||||
onFiltersChange?: (next: FilterMap) => void;
|
||||
// Opt into `table-layout: fixed` (default is `auto`). In fixed mode the table width == its
|
||||
// container at every zoom/text-size — the columns with an explicit `width` keep it, and the rest
|
||||
// (the fluid text columns) split what remains and TRUNCATE instead of pushing the table wider than
|
||||
// the viewport. Off by default so existing content-sized tables (playlists, discovery, Plex) are
|
||||
// untouched; the channel manager turns it on so its row never scrolls horizontally out of view.
|
||||
fixedLayout?: boolean;
|
||||
}) {
|
||||
const { t } = useTranslation();
|
||||
const initial = loadPersist(persistKey);
|
||||
@@ -90,17 +111,42 @@ export default function DataTable<T>({
|
||||
const controlledSort = onSortChange !== undefined;
|
||||
const [internalSort, setInternalSort] = useState<SortState>(initial.sort);
|
||||
const sort = controlledSort ? (sortProp ?? null) : internalSort;
|
||||
const [filters, setFilters] = useState<FilterMap>(initial.filters);
|
||||
// Filters are controlled when the caller passes onFiltersChange (it then owns/persists the map);
|
||||
// otherwise the table keeps its own, seeded from and saved to persistKey — symmetric to sort.
|
||||
const controlledFilters = onFiltersChange !== undefined;
|
||||
const [internalFilters, setInternalFilters] = useState<FilterMap>(initial.filters);
|
||||
const filters = controlledFilters ? (filtersProp ?? EMPTY_FILTERS) : internalFilters;
|
||||
const [page, setPage] = useState(initial.page);
|
||||
const [size, setSize] = useState(initial.size ?? pageSize);
|
||||
|
||||
// Commit a whole new filter map through the owner (controlled) or internal state. A ref holds the
|
||||
// latest resolved map so the externalFilter/reset effects can derive from it without taking
|
||||
// `filters` as a dep (which would re-run them on every filter change).
|
||||
const filtersRef = useRef(filters);
|
||||
filtersRef.current = filters;
|
||||
const commitFilters = (next: FilterMap) => {
|
||||
if (controlledFilters) onFiltersChange!(next);
|
||||
else setInternalFilters(next);
|
||||
};
|
||||
|
||||
// Persist page/size always, but only the sort/filter slices this table actually OWNS — a controlled
|
||||
// caller persists those itself, so writing them here would just leave a stale, never-read duplicate.
|
||||
useEffect(() => {
|
||||
if (persistKey) localStorage.setItem(persistKey, JSON.stringify({ sort, filters, page, size }));
|
||||
}, [persistKey, sort, filters, page, size]);
|
||||
if (!persistKey) return;
|
||||
localStorage.setItem(
|
||||
persistKey,
|
||||
JSON.stringify({
|
||||
sort: controlledSort ? null : sort,
|
||||
filters: controlledFilters ? {} : filters,
|
||||
page,
|
||||
size,
|
||||
}),
|
||||
);
|
||||
}, [persistKey, controlledSort, controlledFilters, sort, filters, page, size]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!externalFilter) return;
|
||||
setFilters((f) => ({ ...f, [externalFilter.key]: externalFilter.value }));
|
||||
commitFilters({ ...filtersRef.current, [externalFilter.key]: externalFilter.value });
|
||||
setPage(0);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [externalFilter?.key, externalFilter?.value]);
|
||||
@@ -113,8 +159,9 @@ export default function DataTable<T>({
|
||||
useEffect(() => {
|
||||
if (resetFiltersToken === undefined || resetFiltersToken === resetRef.current) return;
|
||||
resetRef.current = resetFiltersToken;
|
||||
setFilters({});
|
||||
commitFilters({});
|
||||
setPage(0);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [resetFiltersToken]);
|
||||
|
||||
const sorted = sortRows(filterRows(rows, columns, filters), columns, sort);
|
||||
@@ -132,7 +179,7 @@ export default function DataTable<T>({
|
||||
setPage(0);
|
||||
}
|
||||
function applyFilter(key: string, value: string | string[]) {
|
||||
setFilters((f) => ({ ...f, [key]: value }));
|
||||
commitFilters({ ...filters, [key]: value });
|
||||
setPage(0);
|
||||
}
|
||||
|
||||
@@ -184,7 +231,17 @@ export default function DataTable<T>({
|
||||
{(controlsPosition === "top" || controlsPosition === "both") && topControls}
|
||||
{/* Wide screens: table. The wrapper isn't clipped so header filter popovers can overflow. */}
|
||||
<div className="hidden md:block">
|
||||
<table className="w-full border-collapse text-sm">
|
||||
<table className={`w-full border-collapse text-sm ${fixedLayout ? "table-fixed" : ""}`}>
|
||||
{/* In fixed-layout mode a colgroup drives the column widths: those with an explicit
|
||||
`width` keep it, and the unspecified (fluid) columns split the remainder — so the
|
||||
table never exceeds its container and the fluid cells truncate instead. */}
|
||||
{fixedLayout && (
|
||||
<colgroup>
|
||||
{columns.map((col) => (
|
||||
<col key={col.key} style={col.width ? { width: col.width } : undefined} />
|
||||
))}
|
||||
</colgroup>
|
||||
)}
|
||||
<ColumnHead
|
||||
columns={columns}
|
||||
sort={sort}
|
||||
@@ -196,7 +253,7 @@ export default function DataTable<T>({
|
||||
{paged.map((row) => (
|
||||
<tr
|
||||
key={rowKey(row)}
|
||||
className={`border-b border-border/50 hover:bg-card/40 transition ${
|
||||
className={`border-b border-border/50 hover:bg-(--hover) transition ${
|
||||
rowClassName?.(row) ?? ""
|
||||
}`}
|
||||
>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user