diff --git a/CHANGELOG.md b/CHANGELOG.md index 66aea3bd..c3b12ffd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ edit the release notes and re-run the generator. Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading). +## 0.118.0 — 2026-11-08 + +### New + +- Surprise me now leans on the channels and genres you watch, and one pick in three stays a blind draw, so it remains a surprise. +- On a connection too slow for a Plex video, playback waits and goes on where it was, and says how fast the video needs and how fast it arrives. + +### Fixed + +- A 24/7 stream's archive no longer adds years to a list's total: a total names the videos it could not count as "+N without length". The Feed sorted by Surprise me opens faster. +- Plex films the server's GPU cannot decode (VC-1, older DivX or XviD) start in a few seconds instead of about 20. Text over the Plex picture is readable on any frame. + ## 0.117.0 — 2026-11-06 ### New diff --git a/CLAUDE.md b/CLAUDE.md index 4b6b62a3..0d16a1f3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -65,7 +65,7 @@ how it gets built. Concretely: `routes/*` (HTTP only: validation, auth-dependency, serialization) → domain packages `sync/`, `plex/`, `downloads/`, `reader/`, `discover/`, `catalog/`, `youtube/`, `issues/` (shared by routes + scheduler + worker) → flat core (`auth`, `apitokens`, `quota`, `jobruns`, `sysconfig`, `state`, `audit`, `notifications`, `userscope`, `ratelimit`, `bodylimit`, `progress`, `collation`, -`security`, `logs`, `logsink`, `diagnostics`, `debugsession`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.** +`security`, `logs`, `logsink`, `diagnostics`, `debugsession`, `draw`) → `models/db/config`. **Dependency direction is one-way: core never imports routes.** The domain is four **units**, each owning its tables: Capture (`downloads/`), Reader (`reader/`, `discover/`, the playlist sync), Catalog (`catalog/`, the `sync/` ingests) and Plex (`plex/`). A unit's route does no database work: it reads through the unit's `queries`, writes through its `service` (which flushes) and commits. A unit's code reads another unit's tables diff --git a/VERSION b/VERSION index a38b3bd3..12f9c914 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.117.0 +0.118.0 diff --git a/backend/.basedpyright/baseline.json b/backend/.basedpyright/baseline.json index 8af825b4..75d496cf 100644 --- a/backend/.basedpyright/baseline.json +++ b/backend/.basedpyright/baseline.json @@ -185,118 +185,6 @@ "lineCount": 1 } }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, - { - "code": "reportArgumentType", - "range": { - "startColumn": 50, - "endColumn": 56, - "lineCount": 1 - } - }, { "code": "reportArgumentType", "range": { @@ -364,25 +252,7 @@ } } ], - "./app/downloads/adapters.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 30, - "endColumn": 38, - "lineCount": 1 - } - } - ], "./app/downloads/backfill.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 34, - "endColumn": 38, - "lineCount": 1 - } - }, { "code": "reportReturnType", "range": { @@ -413,14 +283,6 @@ } ], "./app/downloads/pipeline.py": [ - { - "code": "reportArgumentType", - "range": { - "startColumn": 34, - "endColumn": 38, - "lineCount": 1 - } - }, { "code": "reportAttributeAccessIssue", "range": { diff --git a/backend/alembic/versions/0117_videos_length_sort_indexes.py b/backend/alembic/versions/0117_videos_length_sort_indexes.py new file mode 100644 index 00000000..7d1f1e30 --- /dev/null +++ b/backend/alembic/versions/0117_videos_length_sort_indexes.py @@ -0,0 +1,76 @@ +"""videos: the duration sorts ride an index on the video's length, not on the raw column + +Revision ID: 0117_videos_length_sort_indexes +Revises: 0116_followed_explored_channels +Create Date: 2026-11-07 + +A YouTube length past a day is not a length (`catalog.models.LENGTH_MAX_S`): a 24/7 stream's archive +reports the years it has run. The feed's duration sorts now order by `Video.length_seconds`, which +is NULL for such a video, so "longest first" no longer opens on a stream archive. Migration 0096's +two duration indexes are on the raw column and no longer match that ORDER BY; they are replaced by +the same indexes over the length expression. The expression is spelled out here as SQLAlchemy emits +it, with the limit as a literal (an index matches a query's constant, never a bound parameter); +tests/test_feed_sort_indexes.py fails if the two drift apart. + +CONCURRENTLY (autocommit_block), re-runnable, as in 0096. +""" + +from alembic import op +from sqlalchemy import text + +revision: str = "0117_videos_length_sort_indexes" +down_revision: str | None = "0116_followed_explored_channels" +branch_labels: str | None = None +depends_on: str | None = None + +# Frozen with this migration: `catalog.models.LENGTH_MAX_S` is 24 h. +LENGTH = "(CASE WHEN (duration_seconds > 86400) THEN NULL ELSE duration_seconds END)" + +NEW = { + "ix_videos_length_desc": f"{LENGTH} DESC NULLS LAST, id", + "ix_videos_length_asc": f"{LENGTH} ASC NULLS LAST, id", +} +OLD = { + "ix_videos_duration_desc": "duration_seconds DESC NULLS LAST, id", + "ix_videos_duration_asc": "duration_seconds ASC NULLS LAST, id", +} + + +def _valid(name: str) -> bool | None: + """The index's `indisvalid`, or None when there is no index of that name.""" + row = ( + op.get_bind() + .execute( + text( + "SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid " + "WHERE c.relname = :name" + ), + {"name": name}, + ) + .first() + ) + return None if row is None else bool(row[0]) + + +def _build(name: str, columns: str) -> None: + """CREATE INDEX CONCURRENTLY, re-runnable: an INVALID leftover of a failed build is dropped first + (see 0096 `_build`).""" + if _valid(name) is False: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") + op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} ON videos ({columns})") + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + for name, columns in NEW.items(): + _build(name, columns) + for name in OLD: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + for name, columns in OLD.items(): + _build(name, columns) + for name in NEW: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") diff --git a/backend/alembic/versions/0118_videos_shuffle_key.py b/backend/alembic/versions/0118_videos_shuffle_key.py new file mode 100644 index 00000000..683b5a76 --- /dev/null +++ b/backend/alembic/versions/0118_videos_shuffle_key.py @@ -0,0 +1,40 @@ +"""videos: a random key per video, so a seeded shuffle walks an index instead of sorting the scope + +Revision ID: 0118_videos_shuffle_key +Revises: 0117_videos_length_sort_indexes +Create Date: 2026-11-07 + +The feed's Shuffle sort and the Home surprise ordered the whole feed scope by md5(id || seed) on every +call: ~350 ms on prod for a scope of ~213k videos, and a Shuffle page deeper in the list cost more +(767 ms on page 7 on localdev, against 4 ms for Newest). Each video now carries a random number drawn +once (`Video.shuffle_key`); a seed picks a point on that ring and the draw reads the index from there +(`feedquery.shuffle_walk`). + +One ADD COLUMN with a volatile default rewrites the table and fills every row with its own value. On +localdev (278k videos) that took 28 s, against 84 s for an UPDATE of a nullable column, so the rewrite +it is: the table is locked for that time, and a deploy has the app down while it migrates anyway. +The index is (shuffle_key, id): the walk's keyset cursor compares the pair. +""" + +from alembic import op + +revision: str = "0118_videos_shuffle_key" +down_revision: str | None = "0117_videos_length_sort_indexes" +branch_labels: str | None = None +depends_on: str | None = None + + +def upgrade() -> None: + op.execute( + "ALTER TABLE videos ADD COLUMN IF NOT EXISTS shuffle_key double precision " + "NOT NULL DEFAULT random()" + ) + with op.get_context().autocommit_block(): + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_shuffle_key") + op.execute("CREATE INDEX CONCURRENTLY ix_videos_shuffle_key ON videos (shuffle_key, id)") + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + op.execute("DROP INDEX CONCURRENTLY IF EXISTS ix_videos_shuffle_key") + op.execute("ALTER TABLE videos DROP COLUMN IF EXISTS shuffle_key") diff --git a/backend/alembic/versions/0119_plex_transcode_gpu.py b/backend/alembic/versions/0119_plex_transcode_gpu.py new file mode 100644 index 00000000..1ca8c01c --- /dev/null +++ b/backend/alembic/versions/0119_plex_transcode_gpu.py @@ -0,0 +1,65 @@ +"""plex_items.transcode_hw: the facet says whether the GPU ENCODES, under the measured codec sets + +Measured on prod (2026-11-07): the iGPU has no VC-1 Advanced decode, so a VC-1 film never ran on the +GPU although the facet said it did; VC-1, XviD and DivX now take the CPU-decode → GPU-encode path; and an +HDR film runs on the GPU through the OpenCL tonemap although the facet said software. The rule is +`plex.sync.transcode_hw_capable`; the two codec sets below are pinned to `plex.sync` by +tests/test_plex_playback_class.py. + +Rewritten from the stored codec and bit depth. HDR is not stored, so a 10-bit HW-decodable item is set to +software here and its probe watermark cleared: the bit-depth backfill (`plex.sync._probe_media_batch`, +a bounded batch per Plex sync) reads its HDR flag again and moves an HDR one back to the GPU. +""" + +from typing import Sequence, Union + +from alembic import op + +revision: str = "0119_plex_transcode_gpu" +down_revision: Union[str, None] = "0118_videos_shuffle_key" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +HW_DECODABLE_CODECS = ("h264", "hevc", "h265", "vp9", "mpeg2video") +CPU_DECODE_CODECS = ("vc1", "mpeg4", "msmpeg4v3") + + +def _in(codecs: tuple[str, ...]) -> str: + return ", ".join(f"'{c}'" for c in codecs) + + +def upgrade() -> None: + op.execute( + f""" + UPDATE plex_items + SET transcode_hw = CASE + WHEN lower(codec_video) IN ({_in(HW_DECODABLE_CODECS)}) + THEN coalesce(video_bit_depth, 8) <= 8 + WHEN lower(codec_video) IN ({_in(CPU_DECODE_CODECS)}) + THEN coalesce(video_bit_depth, 8) <= 8 + ELSE false + END + WHERE playable = 'transcode' + """ + ) + op.execute( + f""" + UPDATE plex_items + SET media_probed_at = NULL + WHERE playable = 'transcode' + AND lower(codec_video) IN ({_in(HW_DECODABLE_CODECS)}) + AND video_bit_depth > 8 + """ + ) + + +def downgrade() -> None: + # The old rule: a decodable codec (VC-1 counted) at 8 bits; HDR and every other codec software. + op.execute( + f""" + UPDATE plex_items + SET transcode_hw = lower(codec_video) IN ({_in(HW_DECODABLE_CODECS + ("vc1",))}) + AND coalesce(video_bit_depth, 8) <= 8 + WHERE playable = 'transcode' + """ + ) diff --git a/backend/app/README.md b/backend/app/README.md index 66c583cc..d79b0c11 100644 --- a/backend/app/README.md +++ b/backend/app/README.md @@ -25,8 +25,8 @@ Three layers, by *who* runs the code — not by feature: background run's durable row, its words in `runstatus`), `notifications`, `userscope`, `ratelimit`, `progress`, `logs` (the one logging configuration), `logsink` (its `log_events` table), `diagnostics` (a user's consented log - level), `debugsession` (a recording that tags its user's lines, E16) and `collation` (A→Z in the - reader's alphabet, E30), … are single-file primitives many layers import. They depend on nothing above them (a core + level), `debugsession` (a recording that tags its user's lines, E16), `collation` (A→Z in the + reader's alphabet, E30) and `draw` (the weighted, seeded draw "Surprise me" deals from), … are single-file primitives many layers import. They depend on nothing above them (a core module must never import a `routes/…` module). **Extraction trigger: the SECOND consumer.** Keep new logic inline in its first caller. The moment a diff --git a/backend/app/catalog/models.py b/backend/app/catalog/models.py index 5ea760b9..45cd783f 100644 --- a/backend/app/catalog/models.py +++ b/backend/app/catalog/models.py @@ -10,16 +10,22 @@ from sqlalchemy import ( Boolean, Computed, DateTime, + Float, ForeignKey, Integer, JSON, String, Text, UniqueConstraint, + case, func, + literal_column, + null, ) from sqlalchemy.dialects.postgresql import TSVECTOR +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.sql.elements import ColumnElement from app.db import Base from app.models import TimestampMixin @@ -114,6 +120,20 @@ class Subscription(Base, TimestampMixin): # hidden from feeds/search and revisited by the refresh job until they settle into a VOD. LIVE_OR_UPCOMING = ("live", "upcoming") +# A YouTube length past a day is not a length. A 24/7 stream's archive (`was_live`) reports the years +# the stream has run (measured on prod, 2026-11-06: 122,962,914 s), and one such video made a Watch +# later total read 44,073 hours. YouTube caps an upload at 12 hours, so nothing real is lost. Such a +# length is unknown wherever a length is used: the API sends none (`Video.length_seconds`), sums leave +# it out (a playlist's counts it as `without_length`), and the duration filter, sort and Home budget never +# match it. The raw `duration_seconds` stays for what it does prove — that the video is not a Short — +# and for the jobs that ask whether YouTube has answered yet (test_video_length holds who reads it). +LENGTH_MAX_S = 24 * 3600 + + +def known_length(seconds: int | None) -> int | None: + """`seconds` as a video's length, or None when it is not one (see LENGTH_MAX_S).""" + return seconds if seconds is not None and seconds <= LENGTH_MAX_S else None + class Video(Base, TimestampMixin): """A YouTube video. Shared across users; per-user state lives elsewhere.""" @@ -141,8 +161,8 @@ class Video(Base, TimestampMixin): ) thumbnail_url: Mapped[str | None] = mapped_column(String(1024)) # The feed's views / duration / title sorts each ride a migration-defined index in their exact - # ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to - # `feedquery.order_by` by test_feed_sort_indexes. + # ORDER BY, one per direction (and per ICU collation for title): migration 0096 (the duration + # pair since 0117, on `length_seconds`), held to `feedquery.order_by` by test_feed_sort_indexes. duration_seconds: Mapped[int | None] = mapped_column(Integer) view_count: Mapped[int | None] = mapped_column(BigInteger) like_count: Mapped[int | None] = mapped_column(BigInteger) @@ -197,6 +217,37 @@ class Video(Base, TimestampMixin): last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) unavailable_reason: Mapped[str | None] = mapped_column(String(24)) + # A random number in [0, 1), drawn once when the row is written: the ring a seeded shuffle walks + # (`feedquery.shuffle_walk`, index ix_videos_shuffle_key on (shuffle_key, id), migration 0118). + shuffle_key: Mapped[float] = mapped_column(Float, nullable=False, server_default=func.random()) + + @hybrid_property + def length_seconds(self) -> int | None: + """The video's length, or None when it is unknown — not fetched yet, or not a length at all + (`LENGTH_MAX_S`). What every reader of a length reads.""" + return known_length(self.duration_seconds) + + @length_seconds.inplace.expression + @classmethod + def _length_seconds_expression(cls) -> ColumnElement[int | None]: + # The limit is a literal, not a bound parameter: the duration sort indexes (migration 0117) + # are built on this very expression, and Postgres matches an index to a query's constant only. + return case( + (cls.duration_seconds > literal_column(str(LENGTH_MAX_S)), null()), + else_=cls.duration_seconds, + ) + + @hybrid_property + def without_length(self) -> bool: + """A video whose length is unknown — live, an archive past a day, or not fetched yet: what a + sum of lengths leaves out and names as "+N without length". Every such video, so the count + and the sum never disagree about what was left out.""" + return self.length_seconds is None + + @without_length.inplace.expression + @classmethod + def _without_length_expression(cls) -> ColumnElement[bool]: + return cls.length_seconds.is_(None) # Weighted full-text search document (DB-generated, never written by the app): title (A) > # creator keywords + search queries (B) > truncated description (C). ts_rank honours the diff --git a/backend/app/catalog/queries.py b/backend/app/catalog/queries.py index 63b2caee..131f08ba 100644 --- a/backend/app/catalog/queries.py +++ b/backend/app/catalog/queries.py @@ -44,7 +44,8 @@ def channel_titles(db: Session, ids: set[str]) -> dict[str, str]: def channel_aggregates(channel_ids: list[str]) -> Select: - """Per-channel stored count, last upload and total duration, in one grouped pass. Every column it + """Per-channel stored count, last upload and total duration (the lengths it knows: a video with + none is left out, `Video.length_seconds`), in one grouped pass. Every column it reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index alone; a test holds that (test_channel_list_index_only).""" return ( @@ -52,7 +53,7 @@ def channel_aggregates(channel_ids: list[str]) -> Select: Video.channel_id, func.count(Video.id), func.max(Video.published_at), - func.coalesce(func.sum(Video.duration_seconds), 0), + func.coalesce(func.sum(Video.length_seconds), 0), ) .where(Video.channel_id.in_(channel_ids)) .group_by(Video.channel_id) diff --git a/backend/app/discover/queries.py b/backend/app/discover/queries.py index c26e8342..d0469a8a 100644 --- a/backend/app/discover/queries.py +++ b/backend/app/discover/queries.py @@ -10,16 +10,25 @@ explore-visible and in-progress for free, and cannot drift from them. The Plex h narrows EVERY rail by the Plex sidebar's `LibraryFilters`, which is right for a surface that HAS a sidebar and wrong for a landing page that does not. -**One seed drives both halves.** The feed orders a shuffle by `md5(id || seed)` so a keyset cursor -stays valid across pages; the Plex surprise rail uses `ORDER BY random()`, which is fine for a single -un-paged draw but cannot be reproduced. A cross-source reroll needs both halves to move together — -otherwise "Mást kérek" re-draws the Plex picks while the YouTube ones stand still — so the Plex side -here takes the feed's seeded-md5 idiom instead of `random()`. +**One seed drives both halves.** The YouTube half starts on the ring of stored random keys at the +seed's point and shuffles that window by `md5(id || seed)`; the Plex surprise rail uses +`ORDER BY random()`, which is fine for a single un-paged draw but cannot be reproduced. A +cross-source reroll needs both halves to move together — otherwise "Mást kérek" re-draws the Plex +picks while the YouTube ones stand still — so the Plex side here orders by the same seeded md5 +instead of `random()`. """ -from sqlalchemy import and_, func, or_, select +from datetime import datetime, timezone + +from sqlalchemy import ( + and_, + func, + or_, + select, +) from sqlalchemy.orm import Session, aliased +from app import draw from app.discover.budget import band from app.feedquery import ( feed_columns, @@ -27,13 +36,14 @@ from app.feedquery import ( not_blocked_cond, order_by, serialize, + shuffle_walk, sort_keys, ) from app.models import User from app.plex.models import PlexItem, PlexShow, PlexState from app.catalog.models import Channel, Video from app.reader.models import PlaylistItem, VideoState -from app.plex import rollup +from app.plex import affinity, rollup from app.plex.cards import display_title, leaf_card, movie_card, show_card from app.plex.query import apply_show_state_filter, content_libs from app.userscope import owned_by @@ -67,7 +77,8 @@ _FEED_BASE: dict = { def _seeded(col, seed: int): - """The feed's deterministic shuffle expression, reused for Plex so one seed moves both halves.""" + """The Home surprise's seeded order (md5 of the key and the seed), the same for Plex so one + seed moves both halves.""" return func.md5(func.concat(col, str(seed))) @@ -112,17 +123,75 @@ def yt_surprise(db: Session, user: User, budget: str, seed: int, limit: int) -> Shorts are opened up only for the shortest band: "I have ten minutes" is the one answer a Short is actually a good response to, and the feed's own default keeps them out otherwise.""" lo, hi = band(budget) - params = dict(_FEED_BASE, min_duration=lo, max_duration=hi, include_shorts=budget == "short") - query, rank_expr = filtered_query(db, user, **params) - keys = sort_keys("shuffle", seed, rank_expr) - # Over-fetch, then drop anything already started. The feed's `unwatched` scope keeps watched and - # hidden out but deliberately keeps in-progress rows in (they belong in the feed). Here they must - # go: the resume tier has already had its chance at them, and it may have passed precisely because - # the REMAINING time did not fit the budget — offering the same item back as a surprise, whose - # runtime does fit, would label a half-finished thing "Meglepetés" and contradict the question. - rows = db.execute(query.order_by(*order_by(keys)).limit(limit * 4)).all() - out = [serialize(r) for r in rows] - return [r for r in out if not r["position_seconds"]][:limit] + params: dict = { + **_FEED_BASE, + "min_duration": lo, + "max_duration": hi, + "include_shorts": budget == "short", + } + query, _rank = filtered_query(db, user, **params) + + # A window of the ring, not the whole scope: the seed's SURPRISE_WINDOW videos read along the + # stored random key (`shuffle_walk`, a millisecond), then shuffled by the seed. Dealing straight + # from the ring would favour for good every video that sits after a wide gap of it (the first + # video past a random point is the one whose gap the point fell into); shuffling the window makes + # each video's chance the window's share, even to within a few percent. A scope no larger than + # the window is the whole scope, in exactly the md5 order the draw always had. + # + # Anything already started is dropped. The feed's `unwatched` scope keeps watched and hidden out + # but deliberately keeps in-progress rows in (they belong in the feed). Here they must go: the + # resume tier has already had its chance at them, and it may have passed precisely because the + # REMAINING time did not fit the budget — offering the same item back as a surprise, whose runtime + # does fit, would label a half-finished thing "Meglepetés" and contradict the question. + window = [ + r + for r in map(serialize, shuffle_walk(db, query, seed, SURPRISE_WINDOW)) + if not r["position_seconds"] + ] + blind = draw.shuffled(window, seed, key=_video_id) + channels = _channel_affinity(db, user) + if not channels: + return blind[:limit] + + def weight(row: dict) -> float: + return 1.0 + CHANNEL_WEIGHT * channels.get(row["channel_id"], 0.0) + + leaning = draw.leaning(window, seed, key=_video_id, weight=weight) + return draw.blend(leaning, blind, seed, limit, key=_video_id) + + +# How many videos of the ring a surprise draws from: wide enough that a channel the user watches has +# a few in it, small enough to read in a millisecond or two. +SURPRISE_WINDOW = 500 + + +def _video_id(row: dict) -> str: + return row["id"] + + +# How much a channel the user watches most outweighs one they never watch (1 + this), and how far back +# the watched history counts: the latest WATCHED_RECENT watches, the recent ones weighing more. +CHANNEL_WEIGHT = 3.0 +WATCHED_RECENT = 300 +WATCHED_HALF_LIFE_DAYS = 90 + + +def _channel_affinity(db: Session, user: User) -> dict[str, float]: + """Each channel the user watched lately, scored 0..1 against the most watched: the sum of its + watches, each weighing half as much every WATCHED_HALF_LIFE_DAYS. Empty with no history.""" + rows = db.execute( + select(Video.channel_id, VideoState.watched_at) + .join(Video, Video.id == VideoState.video_id) + .where(owned_by(VideoState.user_id, user), VideoState.status == "watched") + .order_by(VideoState.watched_at.desc().nullslast()) + .limit(WATCHED_RECENT) + ).all() + now = datetime.now(timezone.utc) + scores: dict[str, float] = {} + for channel_id, watched_at in rows: + r = draw.recency(watched_at, now, WATCHED_HALF_LIFE_DAYS) + scores[channel_id] = scores.get(channel_id, 0.0) + r + return draw.scaled(scores) def yt_video_row(db: Session, user: User, video_id: str) -> dict | None: @@ -289,21 +358,24 @@ def plex_surprise( if not movie_libs: return [] lo, hi = band(budget) - q = ( - select(PlexItem) - .where( - PlexItem.kind == "movie", - PlexItem.library_id.in_(movie_libs), - _unseen_movie_conds(uid), - ) - .order_by(_seeded(PlexItem.rating_key, seed)) - .limit(limit) + q = select(PlexItem).where( + PlexItem.kind == "movie", + PlexItem.library_id.in_(movie_libs), + _unseen_movie_conds(uid), ) if lo is not None: q = q.where(PlexItem.duration_s >= lo) if hi is not None: q = q.where(PlexItem.duration_s <= hi) - items = db.execute(q).scalars().all() + shuffled = _seeded(PlexItem.rating_key, seed) + blind = db.execute(q.order_by(shuffled, PlexItem.id).limit(limit)).scalars().all() + leaning = affinity.for_user(db, uid) + if leaning: + key = draw.weighted_key(draw.seeded_u(PlexItem.rating_key, seed), leaning.weight(PlexItem)) + weighted = db.execute(q.order_by(key, shuffled, PlexItem.id).limit(limit)).scalars().all() + items = draw.blend(list(weighted), list(blind), seed, limit, key=lambda it: it.id) + else: + items = list(blind) states = rollup.item_state_map(db, uid, [it.id for it in items]) return [movie_card(it, states.get(it.id), lang=lang) for it in items] diff --git a/backend/app/downloads/adapters.py b/backend/app/downloads/adapters.py index ee693f6c..1a932fb3 100644 --- a/backend/app/downloads/adapters.py +++ b/backend/app/downloads/adapters.py @@ -14,10 +14,10 @@ Resolution works both before extraction (from source_kind / URL host) and after import re from urllib.parse import urlparse -import yt_dlp from yt_dlp.networking.impersonate import ImpersonateTarget from app.config import settings +from app.downloads import egress from app.downloads.stated import StatedFailure # Facebook's "Cannot parse data": the page yt-dlp got was a wall, not the content. Both the Chrome retry @@ -33,6 +33,9 @@ class SourceAdapter: """Generic source: yt-dlp handles everything and the ref is already a fetchable URL.""" key = "generic" + # SEC-17: every fetch of this source's download goes through the egress gate (`egress`). A + # source's own host list is no guarantee — its pages, redirects and formats can point anywhere. + egress_guarded = True # --- extraction (worker._extract_with_fallback / backfill) --------------------------------- def resolve_url(self, url: str, ydl_opts: dict) -> str: @@ -92,6 +95,9 @@ class SourceAdapter: class YouTubeAdapter(SourceAdapter): key = "youtube" + # Not through the gate: YouTube's fetches go to Google's fixed hosts, and the PO-token provider + # it needs is the sidecar on our internal network, which the gate exists to refuse. + egress_guarded = False # Errors worth retrying with a different player-client set (YouTube per-client flakiness). _RETRIABLE = ( "not a bot", @@ -179,7 +185,7 @@ class FacebookAdapter(SourceAdapter): post_id = facebook_group_post_id(url) if post_id is None or not ydl_opts.get("cookiefile"): return url - with yt_dlp.YoutubeDL(ydl_opts) as ydl: + with egress.open_ydl(ydl_opts) as ydl: raw = ydl.urlopen(url).read(self._POST_PAGE_MAX_BYTES) page = raw.decode("utf-8", "replace") # Measured: a refused or expired cookie gets the login page (~450 KB, a login diff --git a/backend/app/downloads/backfill.py b/backend/app/downloads/backfill.py index a79ed4e2..ca0cc804 100644 --- a/backend/app/downloads/backfill.py +++ b/backend/app/downloads/backfill.py @@ -16,7 +16,6 @@ Run once per environment: from urllib.parse import urlsplit -import yt_dlp from sqlalchemy import select, update from sqlalchemy.orm import Session @@ -25,7 +24,7 @@ from app.config import settings from app.db import SessionLocal from app.downloads import adapters from app.downloads import edit as editmod -from app.downloads import formats, service +from app.downloads import egress, formats, service from app.downloads.models import DownloadJob, MediaAsset from app.models import DISPLAY_TEXT_MAX from app.titles import normalize_title @@ -43,11 +42,15 @@ def extract_meta(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> di def _extract_meta_scoped(url: str, adapter: adapters.SourceAdapter, asset_id: int) -> dict | None: for clients in adapter.client_sets(): opts = formats.build_ydl_opts( - {}, "%(id)s.%(ext)s", lambda d: None, extra=adapter.ydl_opts_extra(clients) + {}, + "%(id)s.%(ext)s", + lambda d: None, + extra=adapter.ydl_opts_extra(clients), + egress=adapter.egress_guarded, ) opts["skip_download"] = True try: - with yt_dlp.YoutubeDL(opts) as ydl: + with egress.open_ydl(opts) as ydl: info = ydl.extract_info(url, download=False) if info: return info diff --git a/backend/app/downloads/egress.py b/backend/app/downloads/egress.py new file mode 100644 index 00000000..472851ba --- /dev/null +++ b/backend/app/downloads/egress.py @@ -0,0 +1,297 @@ +"""The egress gate: every fetch of a user-supplied download goes through here, and this is where the +address is judged — at the moment of the connection, not when the URL was typed (SEC-17). + +`urlguard` judges the URL the user typed, at enqueue and again in the worker. What that cannot see is +every request yt-dlp makes after it: a public host answering 3xx with an internal `Location`, a page +whose formats point inside, a name that resolves public for the check and private for the fetch (DNS +rebinding). So a guarded download runs with yt-dlp's `proxy` set to this gate, a small HTTP proxy on +the worker's loopback. For each request it resolves the host ONCE, refuses the connection if any +address the name has is one the server must not fetch from (`urlguard.is_forbidden`), and otherwise +connects to the very address it vetted — there is no second lookup for a rebinding answer to slip into. + +Measured (yt-dlp 2026.8.19, 2026-11-06) on the worker image, with a logging proxy: the `proxy` option +reaches every path a download takes — the Requests handler, the CurlCFFI one an impersonated request +uses, the native HLS downloader and an ffmpeg HLS download (segments included). ffmpeg itself also +speaks rtmp, rtsp, ftp, tcp and more, which no HTTP proxy covers, so a guarded download keeps only the +formats it fetches over http(s) (`HttpOnlyFormats`, added by `open_ydl`) and fetches HLS natively. + +A refused connection is answered `403` before anything is dialled, so a live internal port and a dead +one read the same; `errors.user_error_text` then stores the one redacted sentence for both. + +Not used for YouTube: those downloads talk to Google's fixed hosts and to the PO-token sidecar, which +lives on the internal network on purpose (`adapters.YouTubeAdapter.egress_guarded`). +""" + +import ipaddress +import select +import socket +import socketserver +import threading +from collections.abc import Callable +from typing import Any, cast +from urllib.parse import urlsplit + +import yt_dlp +from yt_dlp.postprocessor.common import PostProcessor + +from app import logs +from app.downloads import urlguard + +log = logs.get_logger(__name__) + +Resolver = Callable[..., list] +Forbidden = Callable[[ipaddress.IPv4Address | ipaddress.IPv6Address], bool] + +_BLOCKED = b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" +_BAD_GATEWAY = b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" +_BAD_REQUEST = b"HTTP/1.1 400 Bad Request\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" +_MAX_HEAD = 64 * 1024 +# A tunnel with no traffic either way for this long is closed. Above yt-dlp's own socket_timeout +# (30 s, `formats.build_ydl_opts`), so the gate never ends a transfer yt-dlp would still wait for. +_IDLE_S = 120 +# Hop-by-hop headers a proxy must not pass on; `Connection: close` replaces them, so one connection +# carries one request and a reused client connection can never be steered to a second host unvetted. +_HOP = (b"proxy-", b"connection:", b"keep-alive:") + + +def vetted_address( + host: str, port: int, *, resolve: Resolver = socket.getaddrinfo, forbidden: Forbidden +) -> tuple[str, int] | None: + """The address to connect to for `host:port`, or None when it must not be fetched: the name has + no answer, or ANY of its addresses is forbidden (a round-robin answer is not a coin flip). IPv4 + only, as every download is (`source_address` 0.0.0.0 in `formats.build_ydl_opts`).""" + try: + infos = resolve(host, port, socket.AF_INET, socket.SOCK_STREAM) + except (socket.gaierror, UnicodeError, OSError): + # silent-ok: "no answer" is a refusal, and the caller's refusal is the line that is logged. + return None + addrs = [info[4] for info in infos] + if not addrs: + return None + for addr in addrs: + try: + ip = ipaddress.ip_address(addr[0]) + except ValueError: + # silent-ok: an answer that is not an address is refused like a forbidden one, and the + # caller logs the refusal. + return None + if forbidden(ip): + return None + return addrs[0][0], addrs[0][1] + + +def _read_head(sock: socket.socket) -> bytes | None: + buf = b"" + while b"\r\n\r\n" not in buf: + if len(buf) > _MAX_HEAD: + return None + chunk = sock.recv(8192) + if not chunk: + return None + buf += chunk + return buf + + +def _pipe(a: socket.socket, b: socket.socket) -> None: + try: + _relay(a, b) + except OSError: + # silent-ok: one side hung up mid-transfer (yt-dlp closes a connection once it has read a + # file's size, or gives up on a slow host); whichever side cared reports it itself. + return + + +def _relay(a: socket.socket, b: socket.socket) -> None: + socks = [a, b] + while True: + ready, _, _ = select.select(socks, [], [], _IDLE_S) + if not ready: + return + for s in ready: + data = s.recv(65536) + if not data: + return + (b if s is a else a).sendall(data) + + +class _Handler(socketserver.BaseRequestHandler): + def handle(self) -> None: + client: socket.socket = self.request + head = _read_head(client) + if head is None: + return + head_part, _, body = head.partition(b"\r\n\r\n") + lines = head_part.split(b"\r\n") + try: + method, target, version = lines[0].decode("latin1").split(" ", 2) + except ValueError: + # silent-ok: the only client is yt-dlp on this loopback; a malformed request is answered + # 400 and reaches its own error, which the job records. + client.sendall(_BAD_REQUEST) + return + if method == "CONNECT": + host, _, port_s = target.rpartition(":") + host = host.strip("[]") + upstream = self._dial(host, port_s) + if upstream is None: + return + with upstream: + client.sendall(b"HTTP/1.1 200 Connection established\r\n\r\n") + if body: + upstream.sendall(body) + _pipe(client, upstream) + return + u = urlsplit(target) + if u.scheme != "http" or not u.hostname: + client.sendall(_BLOCKED) + return + try: + port = u.port or 80 + except ValueError: + # silent-ok: as for a malformed request line above — answered 400, recorded by the job. + client.sendall(_BAD_REQUEST) + return + upstream = self._dial(u.hostname, str(port)) + if upstream is None: + return + path = (u.path or "/") + (f"?{u.query}" if u.query else "") + headers = [h for h in lines[1:] if not h.lower().startswith(_HOP)] + request = b"\r\n".join( + [f"{method} {path} {version}".encode("latin1"), *headers, b"Connection: close"] + ) + with upstream: + upstream.sendall(request + b"\r\n\r\n" + body) + _pipe(client, upstream) + + def _dial(self, host: str, port_s: str) -> socket.socket | None: + """Connect to the address `host` was vetted at, or answer the client and return None.""" + client: socket.socket = self.request + try: + port = int(port_s) + except ValueError: + # silent-ok: as for a malformed request line above — answered 400, recorded by the job. + client.sendall(_BAD_REQUEST) + return None + gate = cast("_Gate", self.server) + addr = vetted_address(host, port, resolve=gate.resolve, forbidden=gate.forbidden) + if addr is None: + log.warning( + "egress refused a connection to %s:%s", + host, # log-pii-ok: the host a download tried to reach, never its path or query + port, + extra=logs.event("download.egress.refused", port=port), + ) + client.sendall(_BLOCKED) + return None + try: + return socket.create_connection(addr, timeout=_IDLE_S) + except OSError: + # silent-ok: a public host that does not answer; yt-dlp reports it as the job's own + # network failure, which the job records. + client.sendall(_BAD_GATEWAY) + return None + + +class _Gate(socketserver.ThreadingTCPServer): + daemon_threads = True + allow_reuse_address = True + + def __init__(self, resolve: Resolver, forbidden: Forbidden): + self.resolve = resolve + self.forbidden = forbidden + super().__init__(("127.0.0.1", 0), _Handler) + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + def handle_error(self, request, client_address) -> None: + # socketserver's default prints a bare traceback to stderr, outside the log; a connection the + # gate could not finish is a line with its code instead. + log.exception("egress gate dropped a connection", extra=logs.event("download.egress.error")) + + +def start_gate( + *, resolve: Resolver = socket.getaddrinfo, forbidden: Forbidden = urlguard.is_forbidden +) -> _Gate: + """A gate serving on its own thread. The worker uses the one `proxy_url` starts; a test starts + its own, with a resolver and a judgement it controls.""" + gate = _Gate(resolve, forbidden) + threading.Thread(target=gate.serve_forever, name="egress-gate", daemon=True).start() + return gate + + +# The options key `formats.build_ydl_opts` marks a guarded download with; `open_ydl` reads it. yt-dlp +# keeps unknown keys in its params and never acts on them. +GUARDED = "siftlode_egress_guarded" + + +def open_ydl(opts: dict) -> yt_dlp.YoutubeDL: + """The one way this package opens yt-dlp (a contract test holds it): a guarded download also + gets `HttpOnlyFormats`, which yt-dlp takes only as an instance, not through the options dict.""" + # yt-dlp's stub types its params as a TypedDict; these are the plain dict `formats.build_ydl_opts` + # assembles, the same every caller handed yt-dlp before. + ydl = yt_dlp.YoutubeDL(opts) # type: ignore[arg-type] + if opts.get(GUARDED): + ydl.add_post_processor(HttpOnlyFormats(), when="pre_process") + return ydl + + +_gate: _Gate | None = None +_gate_lock = threading.Lock() + + +def proxy_url() -> str: + """This process's gate, started on first use (the worker's jobs share it).""" + global _gate + with _gate_lock: + if _gate is None: + _gate = start_gate() + return _gate.url + + +# The protocols fetched over http(s) through the proxy option — the only ones the gate sees: yt-dlp's +# own fetchers (http, f4m, ism, the DASH segments and their generator, native HLS — which a bare m3u8 +# also gets, as a guarded download sets hls_prefer_native) and ffmpeg's http input (a live stream or +# a clip), which takes the proxy too. Everything else (rtmp, rtsp, mms, ws) ffmpeg or another tool +# would dial directly. +_HTTP_PROTOCOLS = frozenset( + { + "http", + "https", + "f4m", + "ism", + "m3u8", + "m3u8_native", + "http_dash_segments", + "http_dash_segments_generator", + } +) + + +class HttpOnlyFormats(PostProcessor): + """Drops every format a guarded download would fetch past the gate. Runs as `pre_process`, which + yt-dlp re-reads its formats after ("The pre-processors may have modified the formats"), so the + selector never sees them. With nothing left, yt-dlp reports no formats — a refusal, not a leak.""" + + def set_downloader(self, downloader): + # Not the job's postprocessor hooks: they would report this filter as a "processing" step + # before a byte is fetched (yt-dlp attaches them here, once per call). + self._downloader = downloader + + def run(self, information): + # yt-dlp's stub types the info dict as a TypedDict with optional keys; it is a dict here. + info = cast("dict[str, Any]", information) + if info.get("formats"): + info["formats"] = [f for f in info["formats"] if _over_http(f)] + elif info.get("url") and not _over_http(info): + info["url"] = None + return [], information + + +def _over_http(fmt: dict) -> bool: + url = fmt.get("manifest_url") or fmt.get("url") or "" + if urlsplit(url).scheme not in ("http", "https"): + return False + protocol = fmt.get("protocol") or urlsplit(fmt.get("url") or "").scheme + return all(p in _HTTP_PROTOCOLS for p in protocol.split("+")) diff --git a/backend/app/downloads/formats.py b/backend/app/downloads/formats.py index 959753c9..95066371 100644 --- a/backend/app/downloads/formats.py +++ b/backend/app/downloads/formats.py @@ -28,6 +28,7 @@ import urllib.parse from collections.abc import Iterator from app import logs +from app.downloads import egress as egress_gate log = logs.get_logger(__name__) @@ -294,6 +295,8 @@ def build_ydl_opts( extra: dict | None = None, cookiefile: str | None = None, max_bytes: int | None = None, + *, + egress: bool, ) -> dict: """Assemble yt-dlp options for a single download to `outtmpl` (a full path template). @@ -309,7 +312,11 @@ def build_ydl_opts( a file announced larger than it rather than filling the disk and only then being caught by the quota accounting. Its ffmpeg downloader (a `download_ranges` fetch) does not read it, nor does a fragmented download (YouTube's formats: measured, 718 MB fetched with 1 MB left) — so the - pipeline also holds every produced file to what is left by its real size.""" + pipeline also holds every produced file to what is left by its real size. + + `egress` (SEC-17) routes every fetch through the worker's egress gate, which refuses a connection + into our own network, and keeps only the formats fetched over http(s). Required, with no default: + a new caller has to say which kind of download it builds (`SourceAdapter.egress_guarded`).""" s = normalize(spec) h = s["max_height"] opts: dict = { @@ -364,6 +371,13 @@ def build_ydl_opts( if cookiefile: opts["cookiefile"] = cookiefile + # SEC-17. AFTER the adapter merge, like the cookie: an adapter must not be able to point a + # guarded download past the gate. + if egress: + opts["proxy"] = egress_gate.proxy_url() + opts["hls_prefer_native"] = True + opts[egress_gate.GUARDED] = True + if s["mode"] == "a": opts["format"] = "bestaudio/best" opts["postprocessors"].append( diff --git a/backend/app/downloads/pipeline.py b/backend/app/downloads/pipeline.py index 236b162d..cecda260 100644 --- a/backend/app/downloads/pipeline.py +++ b/backend/app/downloads/pipeline.py @@ -17,7 +17,7 @@ from app.downloads import clip as clipmod from app.downloads import edit as editmod from app.downloads import errors as dlerrors from app.downloads import adapters, credentials, formats, mediakind, playable, probe, quota -from app.downloads import backfill +from app.downloads import backfill, egress from app.downloads import service, storage from app.downloads import urlguard from app.downloads.jobdb import ( @@ -394,6 +394,7 @@ def _extract_with_fallback( _make_progress_hook(job_id, asset_id), cookiefile=cookiefile, max_bytes=max_bytes, + egress=adapter.egress_guarded, ), ) last_exc: Exception | None = None @@ -405,6 +406,7 @@ def _extract_with_fallback( extra=adapter.ydl_opts_extra(clients), cookiefile=cookiefile, max_bytes=max_bytes, + egress=adapter.egress_guarded, ) opts["postprocessor_hooks"] = [_make_pp_hook(job_id)] if section is not None: @@ -415,7 +417,7 @@ def _extract_with_fallback( # (Per-ATTEMPT only; a single extract_info that itself runs past the stall timeout is still # killed — acceptable, as socket_timeout + extractor_retries bound one attempt well under it.) try: - with yt_dlp.YoutubeDL(opts) as ydl: + with egress.open_ydl(opts) as ydl: return ydl.extract_info(target, download=True) except yt_dlp.utils.DownloadError as exc: last_exc = exc diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py index e2e80910..97073d99 100644 --- a/backend/app/downloads/service.py +++ b/backend/app/downloads/service.py @@ -153,7 +153,7 @@ def populate_from_catalog(db: Session, asset: MediaAsset) -> bool: return False asset.title = video.title asset.thumbnail_url = video.thumbnail_url - asset.duration_s = video.duration_seconds + asset.duration_s = video.length_seconds if video.published_at: asset.upload_date = video.published_at.date() channel = db.get(Channel, video.channel_id) @@ -337,7 +337,7 @@ def _clip_source_facts(db: Session, spec: dict) -> dict: "title": video.title, "uploader": channel.title if channel else None, "thumbnail_url": video.thumbnail_url, - "length_s": video.duration_seconds, + "length_s": video.length_seconds, } item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"]) if item is None: diff --git a/backend/app/downloads/urlguard.py b/backend/app/downloads/urlguard.py index 854aa732..69e86e8a 100644 --- a/backend/app/downloads/urlguard.py +++ b/backend/app/downloads/urlguard.py @@ -10,22 +10,12 @@ What this closes: a URL whose host resolves to a loopback / private / link-local is refused, at enqueue AND again in the worker before extraction. Both, because DNS can answer differently between the two moments, and the worker is where the socket is actually opened. -What it does NOT close, stated plainly rather than implied, because both need the same fix — gating -the connection itself with a custom socket factory inside yt-dlp, a much larger change than this: - -* DNS rebinding: a host that passes the worker's check and then answers yt-dlp's own resolution - with an internal address. -* HTTP redirects: a public host that answers 3xx with an internal ``Location``. yt-dlp follows it, - and only the URL the user TYPED was ever judged here. An attacker-controlled host or any public - open redirect reaches it, so this is the cheaper of the two. It does NOT usually yield the - response body — a non-media body fails the mp4 remux and stores nothing — but it does hand back - the readout above: the redaction in ``errors.user_error_text`` fires only for the ``network`` - class, so a dead internal port (refused → ``network`` → redacted) is distinguishable from a live - one (an HTTP answer that is not media → ``unsupported_host``/``format_unavailable``, raw text - kept). Tracked as SEC-17. - -The two checks here stop the straightforward case (typing ``http://127.0.0.1:6379`` or an internal -hostname into the box), which is the reachable one. +What this alone cannot see is every request yt-dlp makes AFTER the typed URL: a public host +answering 3xx with an internal ``Location``, and DNS rebinding (a name that passes this check and +then resolves inside for yt-dlp's own lookup). Those are closed one layer down, by `egress`: a guarded +download's every connection goes through a gate that judges the address it actually dials, with +this module's `is_forbidden`. The checks here stay as the early, structured refusal — a user typing +``http://127.0.0.1:6379`` hears why at once instead of from a failed job. Not used for YouTube jobs: those resolve to a fixed public host from an 11-char id, so there is no user-controlled destination to guard. @@ -62,7 +52,7 @@ class BlockedUrl(Exception): self.reason: UrlBlockedReason = reason -def _is_forbidden(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: +def is_forbidden(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: """Every address family we must not open a connection to. Checked explicitly rather than leaning on ``is_private`` alone, whose coverage differs between IPv4 and IPv6.""" # ::ffff:127.0.0.1 is loopback wearing an IPv6 hat — unwrap before judging it. @@ -113,5 +103,5 @@ def check_url(raw: str) -> None: ip = ipaddress.ip_address(addr) except ValueError: # getaddrinfo returned something unparseable; treat as unusable raise BlockedUrl(REASON_PRIVATE) from None - if _is_forbidden(ip): + if is_forbidden(ip): raise BlockedUrl(REASON_PRIVATE) diff --git a/backend/app/draw.py b/backend/app/draw.py new file mode 100644 index 00000000..5f4d850c --- /dev/null +++ b/backend/app/draw.py @@ -0,0 +1,118 @@ +"""A weighted draw that stays a surprise: what "Surprise me" deals from. + +A title's chance to come first is its weight over the pool's (Efraimidis–Spirakis: order by +-ln(1-u)/w, u uniform per row). The weight is 1 + the title's affinity to what the user watches +(`plex.affinity`, `discover.queries`), so a title nobody leans towards still comes up — only less +often. Two things keep it a surprise rather than a ranking: + +- **u is the seed's md5 of the row's key**, the Home surprise's seeded order. With every weight 1 + the order is exactly that md5 order, so a user with no history sees the unweighted draw, and one + seed still moves both sources of the Home hero together. A draw that is not seeded (the Plex + Discover rail) takes u from random(). +- **One slot of three is blind** (`blend`): dealt from the unweighted order, so the same few + favourites cannot fill the rail. + +The YouTube surprise draws over a window of the feed scope rather than all of it (`yt_surprise`): +the window is read in SQL, and `shuffled` / `leaning` order it here with the same u. + +Both affinities weigh a watch by `recency` and score against the strongest with `scaled`, so the two +sources lean by one rule. +""" + +import hashlib +import math +from collections.abc import Callable, Hashable +from datetime import datetime +from typing import Any + +from sqlalchemy import BigInteger, Float, cast, func, literal +from sqlalchemy.dialects.postgresql import BIT +from sqlalchemy.orm import InstrumentedAttribute +from sqlalchemy.sql.elements import ColumnElement + +# 2**32: the first 8 hex digits of an md5 as a fraction of one. +_SPAN = 4294967296.0 + +# A watch with no date (an old import) counts as one this many half-lives ago. +UNDATED_HALF_LIVES = 2 + + +def recency(watched_at: datetime | None, now: datetime, half_life_days: float) -> float: + """A watch's weight in an affinity: 1 now, half as much every `half_life_days`.""" + if watched_at is None: + return 0.5**UNDATED_HALF_LIVES + return 0.5 ** (max(0.0, (now - watched_at).total_seconds() / 86400) / half_life_days) + + +def scaled(scores: dict[str, float]) -> dict[str, float]: + """Each score over the strongest, so the strongest is 1; empty when nothing scored.""" + top = max(scores.values(), default=0.0) + return {k: v / top for k, v in scores.items()} if top else {} + + +def seeded_u( + key: ColumnElement[Any] | InstrumentedAttribute[Any], seed: int +) -> ColumnElement[float]: + """A uniform u in [0, 1) per row, from md5(key || seed) — the order `md5(...)` ascending gives.""" + digest = func.md5(func.concat(key, str(seed))) + head = cast(cast(literal("x") + func.substr(digest, 1, 8), BIT(32)), BigInteger) + return cast(head, Float) / _SPAN + + +def weighted_key(u: ColumnElement[float], weight: ColumnElement[float]) -> ColumnElement[float]: + """The row's place in a weighted draw, ascending: -ln(1-u)/w. Monotonic in u when every w is 1, + so the unweighted order is unchanged.""" + return -func.ln(1.0 - u) / weight + + +def random_u() -> ColumnElement[float]: + return func.random() + + +def u_of(key: str, seed: int) -> float: + """`seeded_u` in Python: the same u for the same key and seed.""" + return int(hashlib.md5(f"{key}{seed}".encode()).hexdigest()[:8], 16) / _SPAN + + +def shuffled(items: list, seed: int, key: Callable[[Any], str]) -> list: + """`items` in the seed's md5 order — the order `ORDER BY md5(key || seed)` gives.""" + return sorted(items, key=lambda it: hashlib.md5(f"{key(it)}{seed}".encode()).hexdigest()) + + +def leaning( + items: list, seed: int, key: Callable[[Any], str], weight: Callable[[Any], float] +) -> list: + """`items` in the weighted draw's order (`weighted_key` in Python), ties in the md5 order.""" + order = {key(it): i for i, it in enumerate(shuffled(items, seed, key))} + return sorted( + items, + key=lambda it: (-math.log(1.0 - u_of(key(it), seed)) / weight(it), order[key(it)]), + ) + + +def blend( + weighted: list, blind: list, offset: int, n: int, key: Callable[[Any], Hashable] = lambda x: x +) -> list: + """`n` picks: slot i comes from `blind` when (i + offset) % 3 == 0, else from `weighted`, each + skipping what an earlier slot took (`key` tells two picks apart). A side that runs out leaves the + rest to the other.""" + out: list = [] + taken: set = set() + sources = (iter(weighted), iter(blind)) + + def nxt(source): + for item in source: + if key(item) not in taken: + return item + return None + + for i in range(n): + first, second = (sources[1], sources[0]) if (i + offset) % 3 == 0 else sources + item = nxt(first) + if item is None: + item = nxt(second) + if item is None: + break + taken.add(key(item)) + out.append(item) + return out diff --git a/backend/app/feedquery.py b/backend/app/feedquery.py index c703846c..4138b9ea 100644 --- a/backend/app/feedquery.py +++ b/backend/app/feedquery.py @@ -13,7 +13,19 @@ import binascii import json from datetime import date, datetime, timedelta, timezone -from sqlalchemy import BigInteger, Select, and_, case, cast, false, func, or_, select +from sqlalchemy import ( + BigInteger, + Select, + and_, + case, + cast, + false, + func, + literal, + or_, + select, + tuple_, +) from sqlalchemy.orm import Session, aliased from app.collation import collated @@ -68,7 +80,7 @@ def feed_columns(user: User, state) -> list: Channel.handle.label("channel_handle"), Video.published_at, Video.thumbnail_url, - Video.duration_seconds, + Video.length_seconds.label("length_seconds"), Video.view_count, Video.is_short, Video.live_status, @@ -88,7 +100,7 @@ def serialize(row) -> dict: "channel_url": _channel_url(row.channel_id, row.channel_handle), "published_at": iso(row.published_at), "thumbnail_url": row.thumbnail_url, - "duration_seconds": row.duration_seconds, + "duration_seconds": row.length_seconds, "view_count": row.view_count, "is_short": row.is_short, "live_status": row.live_status, @@ -260,9 +272,9 @@ def filtered_query( # OR across the picked channels; every other filter still applies on top. query = query.where(Video.channel_id.in_(channel_ids)) if min_duration is not None: - query = query.where(Video.duration_seconds >= min_duration) + query = query.where(Video.length_seconds >= min_duration) if max_duration is not None: - query = query.where(Video.duration_seconds <= max_duration) + query = query.where(Video.length_seconds <= max_duration) if max_age_days is not None: cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400 query = query.where(Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)) @@ -503,10 +515,10 @@ def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) -> "views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}], "views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}], "duration_desc": [ - {"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"} + {"expr": Video.length_seconds, "asc": False, "nullable": True, "kind": "num"} ], "duration_asc": [ - {"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"} + {"expr": Video.length_seconds, "asc": True, "nullable": True, "kind": "num"} ], "title": [{"expr": title, "asc": True, "nullable": True, "kind": "str"}], "title_desc": [{"expr": title, "asc": False, "nullable": True, "kind": "str"}], @@ -536,15 +548,17 @@ def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) -> }, pub, ], - # Deterministic per-seed shuffle: the same seed yields the same total order, so - # the cursor stays valid across pages of one shuffled run. + # The ring of stored random keys from the seed's point (`shuffle_walk`): the keys at or after + # it, then the ones before. The same seed yields the same total order, so the cursor stays + # valid across pages; `feed_page` reads it as two index ranges, never as this sort. "shuffle": [ { - "expr": func.md5(func.concat(Video.id, str(seed))), + "expr": case((Video.shuffle_key < shuffle_start(seed), 1), else_=0), "asc": True, "nullable": False, - "kind": "str", - } + "kind": "num", + }, + {"expr": Video.shuffle_key, "asc": True, "nullable": False, "kind": "num"}, ], } # Relevance (search-result ranking) only exists when there's a query to rank against; @@ -700,6 +714,43 @@ class BadCursor(ValueError): """A `cursor` this module did not write (or one from another sort): the caller's bad input.""" +# The golden ratio's fraction: consecutive seeds land far apart on the ring, never next to each other. +_GOLDEN = 0.6180339887498949 + + +def shuffle_start(seed: int) -> float: + """The point in [0, 1) where a seed's shuffle starts on the ring of `Video.shuffle_key`. + + Folded to 32 bits first: an int past a float's range overflowed the product, and one past 2**53 + left the product no fraction to land on. A seed the client draws (under a billion) is unchanged.""" + return ((seed % 2**32) * _GOLDEN) % 1.0 + + +def shuffle_walk( + db: Session, query: Select, seed: int, n: int, after: tuple[int, float, str] | None = None +) -> list: + """Up to `n` rows of `query` in the seed's ring order: shuffle_key from the seed's point upwards, + then from 0 up to that point. `after` is a cursor's (phase, key, id): the walk resumes past it. + + Each arc is an ascending range read along ix_videos_shuffle_key with the scope's filters checked + row by row, so a draw costs the rows it reads, not a sort of the scope — the md5(id || seed) order + it replaced read and sorted every row the filters let through (~350 ms on prod for ~213k).""" + start = shuffle_start(seed) + phase_from, key, last_id = after if after is not None else (0, None, None) + rows: list = [] + for phase in (0, 1): + if phase < phase_from or len(rows) >= n: + continue + arc = Video.shuffle_key >= start if phase == 0 else Video.shuffle_key < start + q = query.where(arc) + if key is not None and phase == phase_from: + q = q.where( + tuple_(Video.shuffle_key, Video.id) > tuple_(literal(key), literal(last_id)) + ) + rows += db.execute(q.order_by(Video.shuffle_key, Video.id).limit(n - len(rows))).all() + return rows + + def feed_page( db: Session, user: User, @@ -734,6 +785,14 @@ def feed_page( k["value"] = v if ranked and window is not None: rows = windowed_relevance_rows(db, query, keys, window, cursor_id, limit + 1) + elif sort == "shuffle": + after = None + if vals is not None and cursor_id is not None: + try: + after = (int(vals[0]), float(vals[1]), str(cursor_id)) + except (TypeError, ValueError, OverflowError) as e: # Infinity, a 400-digit key + raise BadCursor("Invalid feed cursor") from e + rows = shuffle_walk(db, query, seed, limit + 1, after) else: query = query.order_by(*order_by(keys)) if cursor_id is not None: diff --git a/backend/app/plex/affinity.py b/backend/app/plex/affinity.py new file mode 100644 index 00000000..656990d8 --- /dev/null +++ b/backend/app/plex/affinity.py @@ -0,0 +1,110 @@ +"""What a user's Plex history leans towards, as a weight for a surprise draw (`app.draw`). + +Two signals, both already in the database (no TMDB call): +- **genres** of the titles the user watched, the recent ones weighing more (a half-life of + HALF_LIFE_DAYS), each scored 0..1 against the strongest; +- the **TMDB recommendations** stored on those titles (`recommendations`), each scored by the most + recent title that recommends it. + +A title counts once however many of its episodes were watched — a binge of one show would otherwise +outvote every film — and only the RECENT_TITLES latest titles count. A candidate's weight is +1 + GENRE_WEIGHT × (the mean score of its genres) + REC_WEIGHT × (its recommendation score), so a +title with no affinity keeps weight 1 and still comes up. +""" + +import json +from dataclasses import dataclass, field +from datetime import datetime, timezone + +from sqlalchemy import Float, cast, func, literal, literal_column, select +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.orm import Session +from sqlalchemy.sql.elements import ColumnElement + +from app import draw +from app.plex.models import PlexItem, PlexShow, PlexState +from app.userscope import owned_by + +HALF_LIFE_DAYS = 180 +RECENT_TITLES = 200 +GENRE_WEIGHT = 2.0 +REC_WEIGHT = 3.0 + + +@dataclass(frozen=True) +class Affinity: + genres: dict[str, float] = field(default_factory=dict) + movie_recs: dict[str, float] = field(default_factory=dict) + show_recs: dict[str, float] = field(default_factory=dict) + + def __bool__(self) -> bool: + return bool(self.genres or self.movie_recs or self.show_recs) + + def weight(self, model: type[PlexItem] | type[PlexShow]) -> ColumnElement[float]: + """The draw weight of a `model` row (a movie or a show).""" + scores = cast(literal(json.dumps(self.genres)), JSONB) + recs = self.show_recs if model is PlexShow else self.movie_recs + genre = func.jsonb_array_elements_text(model.genres).table_valued("value").alias("g") + genre_aff = ( + select( + func.coalesce( + func.avg(func.coalesce(cast(scores.op("->>")(genre.c.value), Float), 0.0)), + 0.0, + ) + ) + .select_from(genre) + .scalar_subquery() + ) + rec_aff = func.coalesce( + cast(cast(literal(json.dumps(recs)), JSONB).op("->>")(model.guid), Float), 0.0 + ) + return literal_column("1.0") + GENRE_WEIGHT * genre_aff + REC_WEIGHT * rec_aff + + +def for_user(db: Session, uid: int) -> Affinity: + """The user's affinity from their latest watched titles; empty (`not aff`) with no history.""" + # One key per title: a show for an episode, the item for a film. concat() of a NULL show_id is + # just "s", which nullif turns back into NULL so the film's key takes over. + title = func.coalesce( + func.nullif(func.concat("s", PlexItem.show_id), "s"), func.concat("m", PlexItem.id) + ).label("title") + latest = ( + select(title, PlexState.watched_at, PlexItem.id.label("item_id"), PlexItem.show_id) + .join(PlexItem, PlexItem.id == PlexState.item_id) + .where(owned_by(PlexState.user_id, uid), PlexState.status == "watched") + .distinct(title) + .order_by(title, PlexState.watched_at.desc().nullslast()) + .subquery() + ) + rows = db.execute( + select( + latest.c.watched_at, + PlexItem.genres, + PlexItem.recommendations, + PlexShow.genres, + PlexShow.recommendations, + ) + .select_from(latest) + .join(PlexItem, PlexItem.id == latest.c.item_id) + .outerjoin(PlexShow, PlexShow.id == latest.c.show_id) + .order_by(latest.c.watched_at.desc().nullslast()) + .limit(RECENT_TITLES) + ).all() + if not rows: + return Affinity() + + now = datetime.now(timezone.utc) + genres: dict[str, float] = {} + movie_recs: dict[str, float] = {} + show_recs: dict[str, float] = {} + for watched_at, item_genres, item_recs, show_genres, show_recs_json in rows: + r = draw.recency(watched_at, now, HALF_LIFE_DAYS) + for g in (show_genres if show_genres is not None else item_genres) or []: + genres[g] = genres.get(g, 0.0) + r + for rec in (show_recs_json if show_recs_json is not None else item_recs) or []: + if not rec.get("tmdb_id"): + continue + bucket = show_recs if rec.get("media_type") == "tv" else movie_recs + guid = f"tmdb:{rec['tmdb_id']}" + bucket[guid] = max(bucket.get(guid, 0.0), r) + return Affinity(draw.scaled(genres), draw.scaled(movie_recs), draw.scaled(show_recs)) diff --git a/backend/app/plex/discover.py b/backend/app/plex/discover.py index ac9ed64a..11461e4e 100644 --- a/backend/app/plex/discover.py +++ b/backend/app/plex/discover.py @@ -22,7 +22,8 @@ from sqlalchemy.orm import Session from app.collation import sorted_words from app.discover.queries import RAIL_LIMIT from app.models import User -from app.plex import rollup +from app import draw +from app.plex import affinity, rollup from app.plex.cards import SHOW_NEW, display_title, leaf_card, movie_card, show_card from app.plex.models import PlexItem, PlexShow, PlexState from app.plex.query import ( @@ -399,22 +400,25 @@ def discover_page(db: Session, user: User, scope: str, lang: str, f: LibraryFilt } -def _draw(db: Session, stmt, key_col, per: int, exclude: list[str]) -> list: +def _draw(db: Session, stmt, key_col, per: int, exclude: list[str], weight=None) -> list: """`per` random rows of `stmt`, the previous draw's (`exclude`) left out while the pool allows: a reshuffle over a small filtered pool kept dealing the same few titles. What the fresh rows leave - short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail.""" - fresh = ( - db.execute(stmt.where(key_col.not_in(exclude)).order_by(func.random()).limit(per)) - .scalars() - .all() - if exclude - else [] - ) + short is topped up from the excluded ones, so a pool smaller than a draw still fills the rail. + With a `weight` (`plex.affinity`) the draw leans on it, every third slot left blind (`app.draw`).""" + + def pick(q, n: int) -> list: + blind = db.execute(q.order_by(func.random()).limit(n)).scalars().all() + if weight is None: + return list(blind) + key = draw.weighted_key(draw.random_u(), weight) + leaning = db.execute(q.order_by(key).limit(n)).scalars().all() + return draw.blend(list(leaning), list(blind), 1, n, key=lambda r: r.rating_key) + + fresh = pick(stmt.where(key_col.not_in(exclude)), per) if exclude else [] if len(fresh) == per: - return list(fresh) + return fresh taken = [key_col.not_in([r.rating_key for r in fresh])] if fresh else [] - rest = db.execute(stmt.where(*taken).order_by(func.random()).limit(per - len(fresh))).scalars() - return [*fresh, *rest.all()] + return [*fresh, *pick(stmt.where(*taken), per - len(fresh))] def _kept( @@ -474,6 +478,8 @@ def surprise( if keep: return {"scope": scope, "items": _kept(db, user.id, movie_ids, show_ids, keep, lang)} per = min(limit, SURPRISE_PER_KIND) + # What the user's history leans towards; with none, every title weighs the same (a plain draw). + leaning = affinity.for_user(db, user.id) movies = [] if movie_ids: # Exclude movies this user has watched. @@ -501,6 +507,7 @@ def surprise( PlexItem.rating_key, per, exclude, + leaning.weight(PlexItem) if leaning else None, ) shows = [] if show_ids: @@ -523,6 +530,7 @@ def surprise( PlexShow.rating_key, per, exclude, + leaning.weight(PlexShow) if leaning else None, ) mcards = _movie_cards(db, user.id, movies, lang) scards = _show_cards(db, user.id, shows, lang) diff --git a/backend/app/plex/models.py b/backend/app/plex/models.py index 7b39d04a..24808ff0 100644 --- a/backend/app/plex/models.py +++ b/backend/app/plex/models.py @@ -227,16 +227,17 @@ class PlexItem(Base, TimestampMixin, UpdatedAtMixin): playable: Mapped[str | None] = mapped_column(String(12), index=True) probed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) # Transcode HW-capability facet (the playback-class filter). NULL for non-transcode items; for a - # `playable="transcode"` item: True = the prod iGPU can hardware-transcode it (codec is HW-decodable - # AND the source is 8-bit SDR), False = software libx264 only (AV1/XviD, or a 10-bit/HDR source that - # breaks the VAAPI nv12 pipeline — see plex/stream.py). Derived by `plex.sync.transcode_hw_capable`: - # set codec-only at sync (bit depth unknown from the cheap listing), then refined by the bit-depth - # backfill (`_probe_media_batch`) which downgrades a 10-bit/HDR HEVC to False. The one filterable - # home of the invariant `_want_hw`'s source gate computes at play time — kept in sync via the shared - # classifier so the catalog facet and the runtime encoder choice never disagree by codec. + # `playable="transcode"` item: True = the prod iGPU ENCODES it (a HW-decodable 8-bit SDR source, a + # true-HDR one through the OpenCL tonemap, or an 8-bit source in a codec the CPU decodes for it — + # VC-1, XviD, DivX), False = software libx264 (AV1, or a 10-bit SDR source — see plex/stream.py). + # Derived by `plex.sync.transcode_hw_capable`: set codec-only at sync (bit depth unknown from the + # cheap listing), then refined by the bit-depth backfill (`_probe_media_batch`), which downgrades a + # 10-bit SDR HEVC to False. The filterable twin of the rule `_select_encode` applies at play time; + # the two share their codec sets, not their code, so a change to one tree is a change to both. transcode_hw: Mapped[bool | None] = mapped_column(Boolean, index=True) # Video bit depth (8/10/12…) from the Plex media Stream, filled by the bit-depth backfill. NULL = - # not yet probed. Surfaced on the info page (e.g. "HEVC · 10-bit") and feeds `transcode_hw`. + # not yet probed. Surfaced on the info page (e.g. "HEVC · 10-bit") and feeds `transcode_hw` (a + # 10-bit SDR source drops to software; an HDR one keeps the GPU through the tonemap). video_bit_depth: Mapped[int | None] = mapped_column(Integer) # Watermark for the bit-depth backfill: when this item's media Stream was last probed for # bit-depth/HDR. NULL = never (the backfill's work-list). Only HW-decodable transcode items are diff --git a/backend/app/plex/stream.py b/backend/app/plex/stream.py index 8e25bbdb..42e31e67 100644 --- a/backend/app/plex/stream.py +++ b/backend/app/plex/stream.py @@ -4,9 +4,10 @@ Playback is from the LOCAL physical file. Browser-compatible files (playable="di raw with HTTP range requests. Everything else that is h264 video (playable="remux") is remuxed on the fly to HLS with **video stream-copy** (cheap, I/O-bound — no video re-encode) and audio to AAC when needed. Non-h264 video (playable="transcode": HEVC/VP9/AV1/MPEG-2/XviD…) needs a full re-encode -to H.264: on the prod host that runs on the Intel iGPU via VAAPI (`h264_vaapi`, near-zero CPU), with -software libx264 as the automatic fallback where the render node is absent or the source codec isn't -hardware-decodable. Same HLS session/seek machinery either way — only the video encoder differs. +to H.264: on the prod host that is encoded on the Intel iGPU via VAAPI (`h264_vaapi`) — decoded there +too when it can, else decoded on the CPU and uploaded (VC-1, XviD, DivX) — with software libx264 as +the fallback where the render node is absent, for a source neither GPU route takes (AV1, 10-bit SDR), +and when a GPU session produces nothing. Same HLS session/seek machinery either way. Seek model (Jellyfin-style): one ffmpeg session per item, started at a given offset via `-ss` (fast keyframe seek). A seek beyond the generated region restarts the session at the new offset, @@ -35,7 +36,7 @@ from app.config import settings from app.downloads.edit import encode_thread_cap from app.plex.models import PlexItem from app.plex import paths -from app.plex.sync import HDR_TRANSFERS, HW_DECODABLE_CODECS +from app.plex.sync import CPU_DECODE_CODECS, HDR_TRANSFERS, HW_DECODABLE_CODECS log = logs.get_logger(__name__) _HLS_ROOT = ( @@ -120,6 +121,7 @@ class HlsSession: hw: bool = False, tonemap: bool = False, user_id: int = 0, + cpu_decode: bool = False, ): # Unguessable per-session id, carried in the URLs the client is handed. A seek-restart # replaces the viewer's registered session for the item — and the old player's next request @@ -135,6 +137,7 @@ class HlsSession: self.transcode = transcode # video re-encoded (non-h264 source), not stream-copied self.hw = hw # transcode ran on the VAAPI hardware encoder (else libx264 software) self.tonemap = tonemap # HW HDR/10-bit path: OpenCL tonemap between VAAPI decode and encode + self.cpu_decode = cpu_decode # HW encode of frames the CPU decoded (a codec the GPU can't) self.start_s = start_s # the REQUESTED seek offset # The REAL media start: with `-ss X -c:v copy` ffmpeg can only cut at the nearest earlier # keyframe K<=X, and hls.js zero-bases the timeline to K (so video.currentTime=0 is K, not X). @@ -188,8 +191,11 @@ _HLS_TAIL = [ # * VAAPI hardware (`h264_vaapi`) — the prod default once the iGPU is passed in. A full GPU pipeline # (decode + scale + encode all on-device) offloads the CPU almost entirely, so several concurrent # 1080p transcodes are fine and the API stays responsive. +# * CPU decode → VAAPI encode (`format=nv12,hwupload` + `h264_vaapi`) — a codec the GPU cannot decode +# but whose 8-bit picture it encodes (`CPU_DECODE_CODECS`: VC-1, XviD, DivX). The encode is the +# expensive half, so this still takes most of the load off the CPU. # * libx264 software — the automatic fallback where `/dev/dri` is absent (dev box, or prod before the -# iGPU enablement) or the source codec isn't hardware-decodable (AV1/XviD on Gen 9.5). CPU-bound, so +# iGPU enablement) or the source fits neither GPU route (AV1, a 10-bit SDR source). CPU-bound, so # it reserves a core via `-threads` (see encode_thread_cap). # Everything else — the `-copyts`/`_HLS_TAIL`/segment/seek-restart model and the AAC audio path — is # unchanged, so hls.js, subtitles, multi-audio and seeking keep working exactly as for remux. @@ -198,10 +204,10 @@ _VAAPI_DEVICE = "/dev/dri/renderD128" # here; its presence is the precondition for the HW HDR→SDR tonemap graph (see _opencl_present). _OPENCL_ICD_DIR = Path("/etc/OpenCL/vendors") # Codecs the prod iGPU (UHD 630 / Quick Sync Gen 9.5) can hardware-DECODE — the precondition for the -# full-GPU VAAPI pipeline. A source outside it (AV1, MPEG-4 ASP/XviD…) falls back to software even with -# the render node present, because the GPU can't decode it to feed its own encoder. THE single source is -# plex.sync, so the runtime encoder choice here and the catalog `transcode_hw` facet can't disagree by -# codec; aliased to the old name to keep the call sites below unchanged. +# full-GPU VAAPI pipeline. A source outside it is decoded on the CPU when its codec is in +# `CPU_DECODE_CODECS` (the GPU still encodes), else transcoded in software. THE single source of both +# sets is plex.sync, so the runtime encoder choice here and the catalog `transcode_hw` facet can't +# disagree by codec; aliased to the old name to keep the call sites below unchanged. _HW_DECODABLE = HW_DECODABLE_CODECS # Color-transfer values that mark a true-HDR source (PQ / HLG). A HW-decodable HDR source can't take the # plain scale_vaapi=format=nv12 path (Gen 9.5 VPP refuses it) — it needs the OpenCL tonemap graph @@ -302,16 +308,22 @@ def _opencl_present() -> bool: return False -def _hwaccel_in_opts(hw: bool, tonemap: bool = False) -> list[str]: +def _hwaccel_in_opts(hw: bool, tonemap: bool = False, cpu_decode: bool = False) -> list[str]: """VAAPI decode options, placed before `-i` so the source is decoded (and kept) on the GPU. `-hwaccel_output_format vaapi` keeps frames as GPU surfaces all the way to `h264_vaapi` — no round-trip to system memory. Empty for the software path. The tonemap variant additionally derives an OpenCL device FROM the VAAPI one (`opencl=ocl@va`) so tonemap_opencl can map the decoded surfaces zero-copy (see _venc_opts); it needs named devices, so it - uses `-init_hw_device` + `-hwaccel_device` rather than the plain path's implicit `-vaapi_device`.""" + uses `-init_hw_device` + `-hwaccel_device` rather than the plain path's implicit `-vaapi_device`. + + The CPU-decode variant opens the VAAPI device for the FILTER graph only (`hwupload` needs one) and + asks for no `-hwaccel`: the codec is one the GPU refuses to decode. This exact form is the one + measured on prod with the HLS muxer and `-copyts` (and to the end of the file).""" if not hw: return [] + if cpu_decode: + return ["-init_hw_device", f"vaapi=va:{_VAAPI_DEVICE}", "-filter_hw_device", "va"] if tonemap: return [ "-init_hw_device", @@ -359,6 +371,7 @@ def _venc_opts( preset: str = "veryfast", crf: int = 20, max_height: int = 0, + cpu_decode: bool = False, ) -> list[str]: """The video-encoder args that replace `-c:v copy` for a transcode session (returns the copy args when not transcoding). A re-encode must MINT its own keyframes at the segment boundaries — the copy @@ -385,6 +398,10 @@ def _venc_opts( "h264_vaapi", *force_kf, ] + if hw and cpu_decode: + # The CPU decoded the frames: convert them to the encoder's 8-bit nv12 and upload them to the + # device `-filter_hw_device` named (see _hwaccel_in_opts). + return ["-vf", "format=nv12,hwupload", "-c:v", "h264_vaapi", *force_kf] if hw: # scale_vaapi with format=nv12 downconverts a 10-bit (p010) HEVC source to 8-bit nv12 so # h264_vaapi (8-bit High) can encode it; for an 8-bit source it's a cheap format pass-through. @@ -422,13 +439,18 @@ def _render_node_present() -> bool: return Path(_VAAPI_DEVICE).exists() -def _select_encode(db: DbSession, item: PlexItem, transcode: bool, src: Path) -> tuple[bool, bool]: - """Pick the transcode video pipeline for this session — returns `(hw, tonemap)`: - * `(False, False)` → software libx264 (the fallback: no render node, non-decodable source, or a - 10-bit/HDR source with no OpenCL runtime to tonemap on the GPU). - * `(True, False)` → full-GPU VAAPI (`scale_vaapi=format=nv12` + `h264_vaapi`) — an 8-bit SDR source. - * `(True, True)` → VAAPI decode → OpenCL tonemap → VAAPI encode — a true-HDR (PQ/HLG) source, when - the OpenCL runtime is present (Gen 9.5 VPP can't tonemap natively). +def _select_encode( + db: DbSession, item: PlexItem, transcode: bool, src: Path +) -> tuple[bool, bool, bool]: + """Pick the transcode video pipeline for this session — returns `(hw, tonemap, cpu_decode)`: + * `(False, False, False)` → software libx264 (the fallback: no render node, a source neither GPU + route takes, or a 10-bit/HDR source with no OpenCL runtime to tonemap on the GPU). + * `(True, False, False)` → full-GPU VAAPI (`scale_vaapi=format=nv12` + `h264_vaapi`) — an 8-bit SDR + source the GPU decodes. + * `(True, True, False)` → VAAPI decode → OpenCL tonemap → VAAPI encode — a true-HDR (PQ/HLG) + source, when the OpenCL runtime is present (Gen 9.5 VPP can't tonemap natively). + * `(True, False, True)` → CPU decode → `hwupload` → VAAPI encode — an 8-bit SDR source in a codec + the GPU cannot decode (`CPU_DECODE_CODECS`). A 10-bit *SDR* source is deliberately NOT a `(True, True)` case: it needs a plain 10→8-bit downconvert, not the HDR tone curve (which would visibly darken/flatten an SDR picture), so it drops to software. @@ -438,31 +460,40 @@ def _select_encode(db: DbSession, item: PlexItem, transcode: bool, src: Path) -> backstops a session that produces nothing — but still routing a true-HDR source through the tonemap graph so it doesn't hit the scale_vaapi wall); "auto" (default) → HW only when the render node is present AND the iGPU can hardware-decode the source, with tonemap chosen only for a true-HDR (PQ/HLG) - source that has an OpenCL runtime, while a 10-bit SDR source drops to software.""" + source that has an OpenCL runtime, while a 10-bit SDR source drops to software. In both "on" and + "auto" a codec the GPU is known not to decode goes the CPU-decode route: asking the GPU to decode it + only bought the fallback's wasted second before libx264.""" + sw = (False, False, False) if not transcode: - return (False, False) + return sw mode = sysconfig.get_str(db, "plex_transcode_hw") if mode == "off" or not _render_node_present(): - return (False, False) + return sw + codec = (item.codec_video or "").lower() if mode == "on": + if codec in CPU_DECODE_CODECS: + return (True, False, True) # Trust the admin: HW wherever the render node exists, skipping the decodability gate (the runtime # backstop catches a miss). Only a true-HDR (PQ/HLG) source needs the tonemap graph — a 10-bit SDR # source takes the plain scale path here, NOT the HDR tone curve. Probe only when OpenCL exists # (short-circuit), so "on" stays probe-free where tonemap can't apply anyway. tonemap = _opencl_present() and _probe_video_kind(src) == _VK_HDR - return (True, tonemap) + return (True, tonemap, False) # auto - if (item.codec_video or "").lower() not in _HW_DECODABLE: - return (False, False) + if codec in CPU_DECODE_CODECS: + # The upload converts to 8-bit nv12: only an 8-bit SDR picture comes through it unchanged. + return (True, False, True) if _probe_video_kind(src) == _VK_HWSAFE else sw + if codec not in _HW_DECODABLE: + return sw kind = _probe_video_kind(src) if kind == _VK_HWSAFE: - return (True, False) # 8-bit SDR → full-GPU scale path + return (True, False, False) # 8-bit SDR → full-GPU scale path if kind == _VK_HDR: # PQ/HLG: HW OpenCL tonemap when the runtime is present, else software libx264. - return (True, True) if _opencl_present() else (False, False) + return (True, True, False) if _opencl_present() else sw # _VK_SDR10 — 10-bit SDR: a plain 10→8-bit downconvert, done exactly on the CPU. Never the HDR tone # curve (it would alter an SDR picture), and off the unverified scale_vaapi 10-bit path — software. - return (False, False) + return sw def _want_hw(db: DbSession, item: PlexItem, transcode: bool, src: Path) -> bool: @@ -471,16 +502,17 @@ def _want_hw(db: DbSession, item: PlexItem, transcode: bool, src: Path) -> bool: return _select_encode(db, item, transcode, src)[0] -# CPU-nice increment for the SOFTWARE transcode child. The Plex ffmpeg is a Popen in the long-lived API -# process (which must NOT itself be niced), so — unlike the download worker's _deprioritize (run once in -# its own main()) — we lower priority per-child via preexec_fn in the fork. The GPU path needs none of -# this. Reuses the download child's nice default so all co-located re-encoders share one handicap. +# CPU-nice increment for a transcode child that works the CPU: a software encode, or a CPU decode +# feeding the GPU encoder. The Plex ffmpeg is a Popen in the long-lived API process (which must NOT +# itself be niced), so — unlike the download worker's _deprioritize (run once in its own main()) — we +# lower priority per-child via preexec_fn in the fork. The full-GPU path needs none of this. Reuses +# the download child's nice default so all co-located re-encoders share one handicap. _SW_TRANSCODE_NICE = settings.download_child_nice def _sw_child_preexec() -> None: - """Runs in the forked child (POSIX) before exec: renice this software-transcode ffmpeg down so it - can't starve the co-located API / Postgres / playback. Best-effort — a failed renice must not fail + """Runs in the forked child (POSIX) before exec: renice this CPU-bound transcode ffmpeg down so + it can't starve the co-located API / Postgres / playback. Best-effort — a failed renice must not fail playback.""" try: os.nice(_SW_TRANSCODE_NICE) @@ -540,13 +572,15 @@ def _ffmpeg_cmd( crf: int = 20, max_height: int = 0, loudnorm: bool = False, + cpu_decode: bool = False, ) -> list[str]: # For a transcode session the video is re-encoded (VAAPI or libx264) instead of stream-copied; the # hwaccel decode options go before -i, the encoder args replace `-c:v copy` below. Everything else is # identical to remux. `tonemap` is the HDR/10-bit HW variant (OpenCL tonemap between VAAPI decode and # encode) — it needs matching decode-side (named devices) and encode-side (the OpenCL filter graph) opts. - venc = _venc_opts(transcode, hw, tonemap, preset, crf, max_height) - args = ["ffmpeg", "-nostdin", "-loglevel", "error", *_hwaccel_in_opts(hw, tonemap)] + # `cpu_decode` is the HW encode of CPU-decoded frames: a device for the filter graph, no -hwaccel. + venc = _venc_opts(transcode, hw, tonemap, preset, crf, max_height, cpu_decode) + args = ["ffmpeg", "-nostdin", "-loglevel", "error", *_hwaccel_in_opts(hw, tonemap, cpu_decode)] # Keep ORIGINAL timestamps on the output so the first segment carries the true absolute PTS of the # keyframe ffmpeg actually seeked to. hls.js still zero-bases playback to that PTS; we read it back # (see _probe_first_pts) to learn the real content offset. Without -copyts the segment PTS is @@ -800,7 +834,11 @@ def start_session( return directory / segment_name(i, _VIDEO_VARIANT if variants else None) def _spawn( - hw: bool, tonemap: bool = False, *, replace_only: "HlsSession | None" = None + hw: bool, + tonemap: bool = False, + cpu_decode: bool = False, + *, + replace_only: "HlsSession | None" = None, ) -> "HlsSession | None": # `replace_only` makes the supersede identity-aware for the runtime fallback: the whole # check-and-replace happens under ONE _lock, so a newer same-key session (a concurrent @@ -815,10 +853,12 @@ def start_session( preset = sysconfig.get_str(db, "plex_transcode_preset") crf = sysconfig.get_int(db, "plex_transcode_crf") max_height = sysconfig.get_int(db, "plex_transcode_max_height") - # Nice the SOFTWARE transcode child (not the HW path, and not remux) so a CPU-bound re-encode - # can't starve the co-located API/playback. preexec_fn runs in the forked child (POSIX); on a - # platform without fork (the API always runs in a Linux container in prod) it's simply omitted. - preexec = _sw_child_preexec if (transcode and not hw and os.name == "posix") else None + # Nice the child that works the CPU — a SOFTWARE transcode, or a CPU decode feeding the GPU + # encoder (not the full-GPU path, and not remux) — so it can't starve the co-located + # API/playback. preexec_fn runs in the forked child (POSIX); on a platform without fork (the API + # always runs in a Linux container in prod) it's simply omitted. + cpu_bound = transcode and (not hw or cpu_decode) + preexec = _sw_child_preexec if (cpu_bound and os.name == "posix") else None doomed: list[tuple[HlsSession, Path | None]] = [] # Finished even when the new session fails to start (no space, no ffmpeg, a DB error): what # was moved aside under the lock is otherwise left for a restart's sweep. @@ -851,6 +891,7 @@ def start_session( crf=crf, max_height=max_height, loudnorm=loudnorm, + cpu_decode=cpu_decode, ), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, @@ -866,6 +907,7 @@ def start_session( hw=hw, tonemap=tonemap, user_id=user_id, + cpu_decode=cpu_decode, ) _sessions[reg] = s # The one line that says which pipeline a session actually got. Everything downstream @@ -879,6 +921,7 @@ def start_session( transcode=transcode, hw=hw, tonemap=tonemap, + cpu_decode=cpu_decode, variants=len(variants or ()), start_s=round(start_s, 1), ), @@ -937,7 +980,8 @@ def start_session( if not wait_for(_seg(i), timeout=_WARMUP_SEG_TIMEOUT_S, proc=s.proc): break log.info( - "plex hls session start key=%s req=%.1f real=%.3f multi=%s audio=%s transcode=%s hw=%s tonemap=%s", + "plex hls session start key=%s req=%.1f real=%.3f multi=%s audio=%s transcode=%s hw=%s" + " tonemap=%s cpu_decode=%s", key, start_s, s.media_start_s, @@ -946,6 +990,7 @@ def start_session( transcode, s.hw, s.tonemap, + s.cpu_decode, ) return s diff --git a/backend/app/plex/sync.py b/backend/app/plex/sync.py index 2af5530b..782d173f 100644 --- a/backend/app/plex/sync.py +++ b/backend/app/plex/sync.py @@ -60,14 +60,20 @@ def classify_playable(vcodec: str | None, acodec: str | None, container: str | N # Video codecs the prod iGPU (UHD 630 / Quick Sync Gen 9.5) can hardware-DECODE — the precondition for # the full-GPU VAAPI transcode pipeline. THE single source of this set: plex.stream imports it here so -# the runtime encoder choice (`_want_hw`) and this catalog facet can never disagree by codec. A source -# outside it (AV1, MPEG-4 ASP/XviD…) is software-only even on the iGPU (it can't decode it to feed its -# own encoder). -HW_DECODABLE_CODECS = frozenset({"h264", "hevc", "h265", "vp9", "vc1", "mpeg2video"}) +# the runtime encoder choice (`_select_encode`) and this catalog facet can never disagree by codec. VC-1 +# is NOT in it: measured on prod, the iHD driver answers "No support for codec vc1 profile 3" (VC-1 +# Advanced, what a Blu-ray remux carries), so every session that asked the GPU to decode one died and +# fell back to libx264. +HW_DECODABLE_CODECS = frozenset({"h264", "hevc", "h265", "vp9", "mpeg2video"}) +# Codecs the iGPU cannot decode but whose picture it can still ENCODE: the CPU decodes, the frames go up +# to the GPU as 8-bit nv12, and h264_vaapi encodes (plex.stream's `cpu_decode` path). Measured on prod: +# three segments in ~5 s for a 1080p VC-1 remux (libx264: ~17 s) and ~1 s for an SD XviD or DivX +# episode. Only codecs that are always 8-bit SDR, and only ones measured; any other codec stays software. +CPU_DECODE_CODECS = frozenset({"vc1", "mpeg4", "msmpeg4v3"}) # Color-transfer values marking an HDR source (PQ / HLG). THE single source of this set: plex.stream # (the tonemap choice) and downloads.clip (a clip's tonemap) import it here, so a transfer added for -# one is tonemapped by all. A 10-bit/HDR source breaks the VAAPI scale=nv12 path on Gen 9.5 (see -# plex.stream) → software transcode, so it counts as NOT HW-capable. +# one is tonemapped by all. An HDR source breaks the plain VAAPI scale=nv12 path on Gen 9.5, so it takes +# the OpenCL tonemap graph (see plex.stream) — still the GPU encoder. HDR_TRANSFERS = frozenset({"smpte2084", "arib-std-b67"}) @@ -78,18 +84,24 @@ def transcode_hw_capable( hdr: bool = False, ) -> bool | None: """The stored `PlexItem.transcode_hw` facet: for a `playable="transcode"` item, whether the prod - iGPU can HARDWARE-transcode it (else software libx264 only); None for a non-transcode item. HW needs - a HW-decodable codec AND an 8-bit SDR source — a 10-bit/HDR source (bit_depth>8 or an HDR transfer) - breaks the VAAPI nv12 pipeline and falls back to software. `bit_depth`/`hdr` are unknown at sync - time (the cheap listing carries no Stream), so they default to 8-bit SDR (codec-only) and the - bit-depth backfill refines a 10-bit/HDR HEVC to False later.""" + iGPU ENCODES it ("Transcoded on the hardware encoder"), else software libx264; None for a + non-transcode item. The same answer plex.stream's `_select_encode` gives on prod (render node and + OpenCL runtime present, policy "auto"): + * a HW-decodable codec: 8-bit SDR → the full-GPU path; HDR → the OpenCL tonemap, also on the GPU; + 10-bit SDR → software (an exact downconvert, never the HDR tone curve); + * a CPU-decode codec (`CPU_DECODE_CODECS`) at 8 bits → CPU decode, GPU encode; + * any other codec → software. + `bit_depth`/`hdr` are unknown at sync time (the cheap listing carries no Stream), so they default to + 8-bit SDR and the bit-depth backfill refines a 10-bit SDR HEVC to False later.""" if playable != "transcode": return None - if (vcodec or "").lower() not in HW_DECODABLE_CODECS: - return False - if bit_depth is not None and bit_depth > 8: - return False - return not hdr + codec = (vcodec or "").lower() + deep = bit_depth is not None and bit_depth > 8 + if codec in HW_DECODABLE_CODECS: + return hdr or not deep + if codec in CPU_DECODE_CODECS: + return not deep and not hdr + return False def _video_stream_props(meta: dict) -> tuple[int | None, bool]: @@ -266,8 +278,9 @@ def sync(db: Session) -> dict: # a Plex shape change, a DB error or our own bug, and `%s` told them apart badly. log.exception("the Plex people-enrich batch failed") # Probe a bounded batch of HW-decodable transcode items for bit-depth/HDR, so the - # playback-class filter's HW/SW split is precise (a 10-bit/HDR HEVC → software) over - # successive runs. Only HW-decodable-codec transcodes are probed — the rest are SW by codec. + # playback-class filter's HW/SW split is precise (a 10-bit SDR HEVC → software) over + # successive runs. Only HW-decodable-codec transcodes are probed — the rest are classed + # by codec alone. try: stats["media_probed"] = _probe_media_batch(db, plex) except Exception: # best-effort; never fail the sync over a backfill @@ -294,15 +307,15 @@ def sync(db: Session) -> dict: # whole library is covered over successive runs (each fetched item is stamped people_enriched_at once). _ENRICH_BATCH = 60 # Same bounded-per-run pattern for the bit-depth probe. Only HW-decodable-codec transcode items are -# probed (typically a small minority — the bulk of transcodes are old codecs that are SW by codec), so +# probed (typically a small minority — the bulk of transcodes are old codecs classed by codec), so # a modest batch covers the work-list in a run or two; a stuck item retries next run (watermark unset). _PROBE_BATCH = 60 def _probe_media_batch(db: Session, plex: PlexClient, limit: int = _PROBE_BATCH) -> int: """Rich-fetch up to `limit` not-yet-probed HW-decodable transcode items, read the video Stream's - bit-depth + HDR, and store `video_bit_depth` + the refined `transcode_hw` (a 10-bit/HDR HEVC → - False/software). Stamps `media_probed_at` on every attempt (success OR a Plex miss with no Stream), + bit-depth + HDR, and store `video_bit_depth` + the refined `transcode_hw` (a 10-bit SDR HEVC → + False/software; an HDR one stays on the GPU through the tonemap). Stamps `media_probed_at` on every attempt (success OR a Plex miss with no Stream), so an item is probed once and not re-fetched every sync; a transient fetch FAILURE leaves the watermark unset to retry. Oldest-id first. Returns how many items were probed this run.""" rows = ( diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index b5e9c026..b8732d68 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -146,12 +146,17 @@ def item_counts(db: Session, ids: list[int]) -> dict[int, int]: ) -def durations(db: Session, ids: list[int]) -> dict[int, int]: - """Each playlist's running time: the sum of its videos' lengths.""" +def durations(db: Session, ids: list[int]) -> dict[int, tuple[int, int]]: + """Each playlist's running time — the sum of its videos' known lengths — and how many videos + it left out for having none (`Video.without_length`).""" return { - pid: int(total or 0) - for pid, total in db.execute( - select(PlaylistItem.playlist_id, func.coalesce(func.sum(Video.duration_seconds), 0)) + pid: (int(total or 0), int(left_out or 0)) + for pid, total, left_out in db.execute( + select( + PlaylistItem.playlist_id, + func.coalesce(func.sum(Video.length_seconds), 0), + func.count().filter(Video.without_length), + ) .join(Video, Video.id == PlaylistItem.video_id) .where(PlaylistItem.playlist_id.in_(ids)) .group_by(PlaylistItem.playlist_id) @@ -191,11 +196,16 @@ def covers(db: Session, ids: list[int]) -> dict[int, str | None]: ) -def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None]: - """One playlist's item count, running time and cover. An outer join, so an empty list still - answers (0, 0, None).""" - count, total = db.execute( - select(func.count(PlaylistItem.id), func.coalesce(func.sum(Video.duration_seconds), 0)) +def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, int, str | None]: + """One playlist's item count, running time, the videos the running time left out for having no + length (`Video.without_length`) and cover. An outer join, so an empty list still answers + (0, 0, 0, None).""" + count, total, left_out = db.execute( + select( + func.count(PlaylistItem.id), + func.coalesce(func.sum(Video.length_seconds), 0), + func.count().filter(Video.without_length), + ) .select_from(PlaylistItem) .outerjoin(Video, Video.id == PlaylistItem.video_id) .where(PlaylistItem.playlist_id == playlist_id) @@ -207,7 +217,7 @@ def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None .order_by(PlaylistItem.position) .limit(1) ) - return count or 0, total or 0, cover + return count or 0, total or 0, left_out or 0, cover def playlist_rows(db: Session, user: User, playlist_id: int) -> list: diff --git a/backend/app/routes/_common.py b/backend/app/routes/_common.py index 6f904dad..32e1028f 100644 --- a/backend/app/routes/_common.py +++ b/backend/app/routes/_common.py @@ -6,9 +6,9 @@ and "fetch a row this user owns or 404" idioms have one definition, one status c one existence-leak policy. """ -from typing import Any, Protocol, TypeVar +from typing import Annotated, Any, Protocol, TypeVar -from fastapi import HTTPException, Request +from fastapi import HTTPException, Query, Request from pydantic import BaseModel, ConfigDict, Field from sqlalchemy.orm import Session @@ -21,6 +21,10 @@ from app.youtube.client import YouTubeError, refusal_reason M = TypeVar("M", bound=Base) +# A shuffle / surprise seed off the query string. The client draws one under a billion; bounded so a +# hand-typed 400-digit one is a 422, not an int no float or bigint can hold further down. +Seed = Annotated[int, Query(ge=-(2**53), le=2**53)] + # Resume-position thresholds of a player that only keeps a resume point (the YouTube player, the # library's): positions below this are "didn't really start" and within this of the end are # "basically finished" — neither is worth storing, so the position is cleared instead (no progress diff --git a/backend/app/routes/discover.py b/backend/app/routes/discover.py index a180254a..ab574095 100644 --- a/backend/app/routes/discover.py +++ b/backend/app/routes/discover.py @@ -23,7 +23,7 @@ from app.discover import pins, rails from app.discover.budget import Budget, DEFAULT_BUDGET from app.models import User from app.plex.cards import norm_lang -from app.routes._common import not_found +from app.routes._common import Seed, not_found router = APIRouter(prefix="/api/discover", tags=["discover"]) @@ -44,7 +44,7 @@ class ReorderIn(BaseModel): @router.get("") def get_discover( budget: Budget = DEFAULT_BUDGET, - seed: int = 0, + seed: Seed = 0, lang: str = "en", user: User = Depends(current_user), db: Session = Depends(get_db), @@ -79,7 +79,7 @@ MAX_EXCLUDE = 32 @router.get("/pick") def get_pick( budget: Budget = DEFAULT_BUDGET, - seed: int = 0, + seed: Seed = 0, exclude: str = "", start: int = 0, lang: str = "en", diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 43581896..260175f9 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -10,6 +10,7 @@ from sqlalchemy.orm.exc import StaleDataError from app import jobruns, quota, scheduler from app.catalog import queries as catalog_queries, service as catalog_service +from app.catalog.models import known_length from app.auth import current_user from app.db import get_db, release_read_connection from app.mailtext import lang_for_user @@ -19,6 +20,7 @@ from app.models import User from app.reader import queries, service, takeout from app.routes._common import ( ProgressIn, + Seed, capped_body, drain_body, kept_position, @@ -88,7 +90,7 @@ def _feed_params( def get_feed( params: dict = Depends(_feed_params), sort: str = "newest", - seed: int = 0, + seed: Seed = 0, limit: int = Query(default=60, le=200), cursor: str | None = None, user: User = Depends(current_user), @@ -287,7 +289,7 @@ def get_video_detail( "channel_title": v.channel.title if v.channel else None, "published_at": iso(v.published_at), "view_count": v.view_count, - "duration_seconds": v.duration_seconds, + "duration_seconds": v.length_seconds, } # The DB path above is free; the off-catalog YouTube lookup below spends a quota unit. @@ -315,8 +317,8 @@ def get_video_detail( "channel_title": snippet.get("channelTitle"), "published_at": snippet.get("publishedAt"), "view_count": int(views) if views is not None else None, - "duration_seconds": parse_iso8601_duration( - items[0].get("contentDetails", {}).get("duration") + "duration_seconds": known_length( + parse_iso8601_duration(items[0].get("contentDetails", {}).get("duration")) ), } diff --git a/backend/app/routes/playlists.py b/backend/app/routes/playlists.py index 5622dd2f..ba84c110 100644 --- a/backend/app/routes/playlists.py +++ b/backend/app/routes/playlists.py @@ -41,11 +41,16 @@ router = APIRouter(prefix="/api/playlists", tags=["playlists"]) # It guards EVERY route that spends under QuotaAction.PLAYLISTS_PULL, not just the bulk one: a # scripted loop over the single-playlist re-pull drains the same shared budget, and limiting one of # two routes with the same quota action is the asymmetric fix this codebase keeps re-learning. -# Deliberately NOT on the push side (`push-plan` / `push`): those run on every push attempt, -# including the no-op "already up to date" path, so a 3-per-10-minute allowance would refuse -# legitimate work. Their exposure is tracked as SEC-18 instead. _playlist_pull_limiter = RateLimiter(max_events=3, window_seconds=600) +# SEC-18, the push side: `push-plan` reads the live list (1 unit a page) and `push` reads and writes +# it, both against the same instance-wide budget. They get a budget of their own, counted in calls +# and far wider than the pull's: a press in the UI is one plan and, unless nothing changed, one push, +# so no person reaches it, while a scripted loop stops at it. A big push is not this limit's job — +# `push` already refuses one the day's remaining budget cannot cover. +PUSH_ALLOWANCE = 20 +_playlist_push_limiter = RateLimiter(max_events=PUSH_ALLOWANCE, window_seconds=600) + def _check_pull_cooldown(user: User) -> None: """Shared gate so the two PLAYLISTS_PULL routes cannot drift apart again.""" @@ -58,6 +63,7 @@ def _check_pull_cooldown(user: User) -> None: PlaylistRefusal = Literal[ "name_required", "not_on_youtube", + "push_cooldown", "quota_exceeded", "sync_all_busy", "sync_busy", @@ -89,10 +95,13 @@ def _video_or_404(db: Session, video_id: str) -> None: def _summary( pl: "Playlist", count: int, - total_duration: int = 0, + duration: tuple[int, int] = (0, 0), has_video: bool | None = None, cover: str | None = None, ) -> dict: + """`duration` is `queries.durations`'s pair: the known running time, and the videos it left out + for having no length — the "+N without length" beside it.""" + total_duration, left_out = duration out = { "id": pl.id, "name": pl.name, @@ -102,6 +111,7 @@ def _summary( "dirty": pl.dirty, "item_count": count, "total_duration_seconds": total_duration, + "without_length": left_out, "cover_thumbnail": cover, # When this list last CHANGED (rename, add/remove/move/reorder) — the rail's # "recently changed first" sort reads it. See `service.touch` for why it isn't automatic. @@ -132,7 +142,7 @@ def list_playlists( _summary( p, counts.get(p.id, 0), - durations.get(p.id, 0), + durations.get(p.id, (0, 0)), (p.id in member) if contains else None, covers.get(p.id), ) @@ -244,9 +254,16 @@ def revert_youtube( raise youtube_refusal(exc, playlist=True) from exc +def _check_push_cooldown(user: User) -> None: + """Shared gate of the two push routes, so they spend one allowance and cannot drift apart.""" + if not _playlist_push_limiter.allow(str(user.id)): + raise _refused(429, "push_cooldown") + + def _pushable(db: Session, user: User, playlist_id: int) -> "Playlist": - """A local playlist the user may push to YouTube (create or update). The built-in - Watch later and read-only YouTube mirrors are never pushable.""" + """A playlist the user may push to YouTube: a local one (created there on its first push) or a + mirror of their own YouTube list (its local edits merged back). Only the built-in Watch later is + never pushable.""" pl = _own_playlist(db, user, playlist_id) if pl.kind == "watch_later": raise _refused(400, "watch_later_not_pushable") @@ -263,6 +280,7 @@ def push_plan( the quota estimate, so the UI can show a confirm. For a linked playlist this reads the live YouTube state (cheap, 1 unit/page). Write scope is required — syncing is a YouTube write.""" pl = _pushable(db, user, playlist_id) + _check_push_cooldown(user) # after the refusal: a list that cannot be pushed spends nothing try: with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH): plan = plan_merge(db, user, pl) @@ -288,6 +306,8 @@ def push( with sync_claim(db, pl.id) as free: if not free: raise _refused(409, "sync_busy") + # Spent only by a push that runs, as Revert's pull: a busy list or Watch later costs nothing. + _check_push_cooldown(user) try: with quota.attribute(user.id, quota.QuotaAction.PLAYLISTS_PUSH): # Read the live playlist ONCE (linked playlists only) and share it with both the @@ -349,8 +369,8 @@ def rename_playlist( raise _refused(400, "name_required") service.rename_playlist(db, pl, name) db.commit() - count, total, cover = queries.playlist_totals(db, pl.id) - return _summary(pl, count, total, cover=cover) + count, total, left_out, cover = queries.playlist_totals(db, pl.id) + return _summary(pl, count, (total, left_out), cover=cover) @router.delete("/{playlist_id}") diff --git a/backend/tests/test_abuse_limits.py b/backend/tests/test_abuse_limits.py index 4333d269..ae30f3dd 100644 --- a/backend/tests/test_abuse_limits.py +++ b/backend/tests/test_abuse_limits.py @@ -125,3 +125,109 @@ def test_the_single_playlist_repull_shares_the_pull_cooldown(db, as_user, make_u r = c.post("/api/playlists/sync-youtube") assert r.status_code == 429 assert r.json()["detail"] == {"code": "sync_cooldown"} + + +def _pushable_list(db, make_user, monkeypatch, plroutes): + """A local (never exported) list and a user with the write scope; the merge itself is stubbed — + the limiter is under test, not the YouTube round trip.""" + from app import auth + from app.reader.models import Playlist + + user = make_user() + pl = Playlist(user_id=user.id, name="Local", source="local") + db.add(pl) + db.commit() + monkeypatch.setattr(auth, "has_write_scope", lambda u: True) + monkeypatch.setattr(plroutes, "plan_merge", lambda d, u, p, live=None: {"units_estimate": 0}) + monkeypatch.setattr(plroutes, "merge_playlist", lambda d, u, p, live=None: {"added": 0}) + return user, pl + + +def test_the_push_side_goes_on_cooldown_after_its_allowance(db, as_user, make_user, monkeypatch): + # SEC-18: push-plan reads the live list and push writes it, both against the instance-wide + # YouTube budget. A press in the UI is one plan + one push, so the allowance is generous — but a + # scripted loop stops at it, with a reason the UI can say. + from app.routes import playlists as plroutes + + user, pl = _pushable_list(db, make_user, monkeypatch, plroutes) + c = as_user(user) + allowance = plroutes.PUSH_ALLOWANCE + codes = [c.get(f"/api/playlists/{pl.id}/push-plan").status_code for _ in range(allowance)] + assert codes == [200] * allowance + r = c.get(f"/api/playlists/{pl.id}/push-plan") + assert r.status_code == 429 + assert r.json()["detail"] == {"code": "playlist", "reason": "push_cooldown"} + + +def test_plan_and_push_draw_on_one_allowance(db, as_user, make_user, monkeypatch): + # The asymmetric fix again, avoided up front: a limit on one of the two routes leaves the same + # drain open next door. + from app.routes import playlists as plroutes + + user, pl = _pushable_list(db, make_user, monkeypatch, plroutes) + c = as_user(user) + for i in range(plroutes.PUSH_ALLOWANCE): + if i % 2: + r = c.get(f"/api/playlists/{pl.id}/push-plan") + else: + r = c.post(f"/api/playlists/{pl.id}/push") + assert r.status_code == 200, (i, r.json()) + r = c.post(f"/api/playlists/{pl.id}/push") + assert r.status_code == 429 + assert r.json()["detail"] == {"code": "playlist", "reason": "push_cooldown"} + + +def test_a_refused_push_spends_no_allowance(db, as_user, make_user, monkeypatch): + # The allowance counts pushes that run: Watch later (never pushable) and a list another sync + # holds are refused before any YouTube call, so they must not leave the next real press at 429. + from contextlib import contextmanager + + from app.reader.models import Playlist + from app.routes import playlists as plroutes + + user, pl = _pushable_list(db, make_user, monkeypatch, plroutes) + wl = Playlist(user_id=user.id, name="Watch later", kind="watch_later") + db.add(wl) + db.commit() + c = as_user(user) + over = plroutes.PUSH_ALLOWANCE + 1 + assert {c.get(f"/api/playlists/{wl.id}/push-plan").status_code for _ in range(over)} == {400} + assert {c.post(f"/api/playlists/{wl.id}/push").status_code for _ in range(over)} == {400} + + @contextmanager + def busy(_db, _id): + yield False + + with monkeypatch.context() as m: + m.setattr(plroutes, "sync_claim", busy) + assert {c.post(f"/api/playlists/{pl.id}/push").status_code for _ in range(over)} == {409} + assert c.post(f"/api/playlists/{pl.id}/push").status_code == 200 + + +def test_a_mirrored_youtube_list_is_pushable(db, as_user, make_user, monkeypatch): + # A mirror of the user's own YouTube list takes local edits (`dirty`) and syncs them back; only + # Watch later is refused. + from app.reader.models import Playlist + from app.routes import playlists as plroutes + + user, _ = _pushable_list(db, make_user, monkeypatch, plroutes) + mirror = Playlist( + user_id=user.id, name="Mine", source="youtube", yt_playlist_id="PLx", dirty=True + ) + db.add(mirror) + db.commit() + assert as_user(user).get(f"/api/playlists/{mirror.id}/push-plan").status_code == 200 + + +def test_the_push_cooldown_is_per_user(db, as_user, make_user, monkeypatch): + from app.routes import playlists as plroutes + + a, pa = _pushable_list(db, make_user, monkeypatch, plroutes) + b, pb = _pushable_list(db, make_user, monkeypatch, plroutes) + ca = as_user(a) + codes = [ + ca.get(f"/api/playlists/{pa.id}/push-plan").status_code + for _ in range(plroutes.PUSH_ALLOWANCE + 1) + ] + assert codes == [200] * plroutes.PUSH_ALLOWANCE + [429] + assert as_user(b).get(f"/api/playlists/{pb.id}/push-plan").status_code == 200 diff --git a/backend/tests/test_clip.py b/backend/tests/test_clip.py index 9f5a10bf..0c74189e 100644 --- a/backend/tests/test_clip.py +++ b/backend/tests/test_clip.py @@ -531,6 +531,9 @@ def test_a_youtube_clip_hands_the_range_to_yt_dlp(monkeypatch, tmp_path): def __enter__(self): return self + def add_post_processor(self, pp, when="post_process"): + pass # the egress format filter; nothing here to filter + def __exit__(self, *_a): return False diff --git a/backend/tests/test_egress.py b/backend/tests/test_egress.py new file mode 100644 index 00000000..19d980bf --- /dev/null +++ b/backend/tests/test_egress.py @@ -0,0 +1,412 @@ +"""SEC-17: the egress gate judges the address a guarded download actually dials. + +Two loopback addresses stand in for the two sides: 127.0.0.2 plays a public host, 127.0.0.1 our own +network. The gate under test refuses exactly 127.0.0.1, so every assertion is about the judgement and +the wiring, with real sockets, a real yt-dlp and a real ffmpeg — no mocked transport. +""" + +import contextlib +import http.server +import ipaddress +import socket +import socketserver +import struct +import threading +import time +from pathlib import Path + +import pytest +import yt_dlp +from yt_dlp.utils import download_range_func + +from app.downloads import adapters, egress, errors, formats + +PUBLIC = "127.0.0.2" +INTERNAL = "127.0.0.1" + + +def _forbid_internal(ip: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool: + return str(ip) == INTERNAL + + +@pytest.fixture +def gate(): + g = egress.start_gate(forbidden=_forbid_internal) + yield g + g.shutdown() + g.server_close() + + +class _Listener: + """A TCP listener that only counts who connected — the internal service under attack.""" + + def __init__(self, host: str = INTERNAL, port: int = 0): + self.hits = 0 + self.sock = socket.socket() + self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.sock.bind((host, port)) + self.sock.listen() + self.port = self.sock.getsockname()[1] + threading.Thread(target=self._serve, daemon=True).start() + + def _serve(self): + while True: + try: + conn, _ = self.sock.accept() + except OSError: + return + self.hits += 1 + with contextlib.suppress(OSError): + conn.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + conn.close() + + def close(self): + self.sock.close() + + +def _public_server(routes: dict, port: int = 0, directory: Path | None = None): + """An HTTP server on the 'public' address: `routes` maps a path to (status, headers); anything + else is served from `directory`.""" + + class H(http.server.SimpleHTTPRequestHandler): + def __init__(self, *a, **k): + super().__init__(*a, directory=str(directory or "."), **k) + + def log_message(self, *a): + pass + + def do_GET(self): + if self.path in routes: + status, headers, body = routes[self.path] + self.send_response(status) + for k, v in headers.items(): + self.send_header(k, v) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + return + super().do_GET() + + srv = socketserver.ThreadingTCPServer((PUBLIC, port), H) + srv.daemon_threads = True + threading.Thread(target=srv.serve_forever, daemon=True).start() + return srv + + +def _guarded_opts(tmp_path: Path, gate_url: str, **extra) -> dict: + opts = formats.build_ydl_opts({}, str(tmp_path / "%(id)s.%(ext)s"), lambda d: None, egress=True) + opts.update(retries=0, fragment_retries=0, extractor_retries=0, socket_timeout=5) + opts["proxy"] = gate_url # the test's gate (it lets 127.0.0.2 through), not the worker's + opts.update(extra) + return opts + + +def _download_error(opts: dict, url: str) -> str: + with pytest.raises(yt_dlp.utils.DownloadError) as exc, egress.open_ydl(opts) as ydl: + ydl.extract_info(url, download=True) + return str(exc.value) + + +# --- the judgement --------------------------------------------------------------------------------- + + +def test_a_name_with_any_forbidden_answer_is_refused(): + def resolve(answers): + return lambda host, port, *a: [(2, 1, 6, "", (ip, port)) for ip in answers] + + assert egress.vetted_address( + "x", 80, resolve=resolve([PUBLIC]), forbidden=_forbid_internal + ) == ( + PUBLIC, + 80, + ) + assert ( + egress.vetted_address( + "x", 80, resolve=resolve([PUBLIC, INTERNAL]), forbidden=_forbid_internal + ) + is None + ) + + def no_answer(*a): + raise socket.gaierror("no such name") + + assert egress.vetted_address("x", 80, resolve=no_answer, forbidden=_forbid_internal) is None + + +def test_the_real_judgement_refuses_our_own_networks(): + # The worker's gate uses urlguard's judgement unchanged — the same one the enqueue check uses. + for addr in ("127.0.0.1", "10.1.2.3", "172.18.0.5", "192.168.1.1", "169.254.169.254"): + assert egress.vetted_address(addr, 80, forbidden=egress.urlguard.is_forbidden) is None, addr + assert egress.vetted_address("93.184.215.14", 80, forbidden=egress.urlguard.is_forbidden) + + +def test_the_gate_dials_the_address_it_vetted_not_a_second_answer(gate): + # DNS rebinding: a name that answers public for the check and private for the fetch. The gate + # resolves once per connection and dials that answer. + public = _public_server({"/": (200, {}, b"public")}) + internal = _Listener(INTERNAL, public.server_address[1]) + calls = [] + + def rebinding(host, port, *a): + calls.append(host) + ip = PUBLIC if len(calls) == 1 else INTERNAL + return [(2, 1, 6, "", (ip, port))] + + gate.resolve = rebinding + try: + with socket.create_connection(("127.0.0.1", gate.server_address[1])) as s: + s.sendall( + f"GET http://rebind.test:{public.server_address[1]}/ HTTP/1.1\r\n" + f"Host: rebind.test\r\n\r\n".encode() + ) + answer = b"".join(iter(lambda: s.recv(4096), b"")) + assert answer.endswith(b"public") + assert calls == ["rebind.test"] + assert internal.hits == 0 + finally: + public.shutdown() + internal.close() + + +def test_a_tunnel_into_the_network_is_refused_before_it_is_dialled(gate): + internal = _Listener() + try: + with socket.create_connection(("127.0.0.1", gate.server_address[1])) as s: + s.sendall(f"CONNECT {INTERNAL}:{internal.port} HTTP/1.1\r\n\r\n".encode()) + answer = s.recv(4096) + assert answer.startswith(b"HTTP/1.1 403") + assert internal.hits == 0 + finally: + internal.close() + + +def test_a_client_that_hangs_up_mid_transfer_is_not_an_error(gate, monkeypatch): + # yt-dlp closes a connection once it has read what it needed; seen live as a traceback the gate + # printed for every such download. + errors_seen = [] + monkeypatch.setattr(gate, "handle_error", lambda *a: errors_seen.append(a)) + public = _public_server({"/big": (200, {}, b"x" * 4_000_000)}) + try: + s = socket.create_connection(("127.0.0.1", gate.server_address[1])) + s.sendall(f"GET http://{PUBLIC}:{public.server_address[1]}/big HTTP/1.1\r\n\r\n".encode()) + s.recv(1024) + s.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + s.close() # a reset, not a polite close + time.sleep(0.5) + assert errors_seen == [] + finally: + public.shutdown() + + +# --- a real yt-dlp through it ---------------------------------------------------------------------- + + +def test_a_redirect_into_the_network_reaches_it_unguarded_and_not_through_the_gate(tmp_path, gate): + internal = _Listener() + public = _public_server( + {"/v.mp4": (302, {"Location": f"http://{INTERNAL}:{internal.port}/v.mp4"}, b"")} + ) + url = f"http://{PUBLIC}:{public.server_address[1]}/v.mp4" + try: + # The hole, as it was: yt-dlp follows the 3xx into our network. + bare = {"outtmpl": str(tmp_path / "bare" / "%(id)s.%(ext)s"), "quiet": True, "retries": 0} + with contextlib.suppress(yt_dlp.utils.DownloadError), yt_dlp.YoutubeDL(bare) as ydl: + ydl.extract_info(url, download=True) + assert internal.hits >= 1 + before = internal.hits + + _download_error(_guarded_opts(tmp_path, gate.url), url) + assert internal.hits == before + finally: + public.shutdown() + internal.close() + + +def test_a_live_and_a_dead_internal_port_leave_the_same_text(tmp_path, gate): + live = _Listener() + dead = _Listener() + dead_port = dead.port + dead.close() + public = _public_server( + { + "/live.mp4": (302, {"Location": f"http://{INTERNAL}:{live.port}/x"}, b""), + "/dead.mp4": (302, {"Location": f"http://{INTERNAL}:{dead_port}/x"}, b""), + } + ) + base = f"http://{PUBLIC}:{public.server_address[1]}" + try: + stored = [] + for name in ("live", "dead"): + msg = _download_error(_guarded_opts(tmp_path / name, gate.url), f"{base}/{name}.mp4") + stored.append(errors.user_error_text("url", msg, errors.classify(msg))) + assert stored[0] == stored[1] + assert live.hits == 0 + finally: + public.shutdown() + live.close() + + +def test_a_format_fetched_past_the_gate_is_never_chosen(tmp_path): + # ffmpeg speaks rtmp, rtsp, tcp… directly, where no HTTP proxy is asked. A guarded download keeps + # only the formats fetched over http(s). + opts = _guarded_opts(tmp_path, "http://127.0.0.1:9") + info = { + "id": "x", + "title": "x", + "extractor": "generic", + "extractor_key": "Generic", + "webpage_url": f"http://{PUBLIC}/page", + "formats": [ + {"format_id": "rtmp", "url": f"rtmp://{INTERNAL}/live", "ext": "flv", "height": 1080}, + {"format_id": "web", "url": f"http://{PUBLIC}/a.mp4", "ext": "mp4", "height": 360}, + ], + } + with egress.open_ydl(opts) as ydl: + chosen = ydl.process_ie_result(dict(info), download=False) + assert chosen["format_id"] == "web" + + only_rtmp = dict(info, formats=[info["formats"][0]]) + with ( + pytest.raises(yt_dlp.utils.YoutubeDLError, match="Requested format is not available"), + egress.open_ydl(opts) as ydl, + ): + ydl.process_ie_result(only_rtmp, download=False) + + +def test_a_format_yt_dlp_fetches_itself_is_kept(): + # f4m, ism and the DASH generator fetch through yt-dlp's own urlopen; a bare m3u8 goes to the + # native HLS fetcher under hls_prefer_native (set for every guarded download). All ask the gate. + kept = [ + {"format_id": p, "url": f"https://{PUBLIC}/{p}", "protocol": p} + for p in ("f4m", "ism", "m3u8", "m3u8_native", "http_dash_segments_generator", "https") + ] + raw = {"format_id": "rtmp", "url": f"rtmp://{PUBLIC}/live", "protocol": "rtmp"} + _, info = egress.HttpOnlyFormats().run({"formats": [*kept, raw]}) + assert [f["format_id"] for f in info["formats"]] == [f["format_id"] for f in kept] + + +def test_a_single_url_result_past_the_gate_loses_its_url(): + _, info = egress.HttpOnlyFormats().run({"url": f"rtmp://{INTERNAL}/live"}) + assert info["url"] is None + _, info = egress.HttpOnlyFormats().run({"url": f"https://{PUBLIC}/a.mp4"}) + assert info["url"] == f"https://{PUBLIC}/a.mp4" + + +def test_the_format_filter_says_no_phase_of_its_own(tmp_path): + # It is a post-processor only to run before the format choice; the job's postprocessor hook + # must not see it (it painted every guarded job "processing" before a byte was fetched). + calls: list[dict] = [] + opts = _guarded_opts(tmp_path, "http://127.0.0.1:9", postprocessor_hooks=[calls.append]) + with egress.open_ydl(opts) as ydl: + ydl.run_all_pps("pre_process", {"id": "x", "formats": []}) + assert calls == [] + + +def test_an_allowed_tunnel_carries_bytes_both_ways(gate): + # The CONNECT path a site served over https takes: a 200, then the bytes the client sent after + # the head (in the same packet here) reach the far end, and its answer comes back. + far = socket.socket() + far.bind((PUBLIC, 0)) + far.listen() + + def echo(): + conn, _ = far.accept() + with conn: + conn.sendall(b"PONG:" + conn.recv(64)) + + threading.Thread(target=echo, daemon=True).start() + try: + with socket.create_connection(("127.0.0.1", gate.server_address[1]), timeout=5) as s: + s.sendall(f"CONNECT {PUBLIC}:{far.getsockname()[1]} HTTP/1.1\r\n\r\nPING".encode()) + got = b"" + while b"PONG:PING" not in got: + chunk = s.recv(4096) + if not chunk: + break + got += chunk + assert got.startswith(b"HTTP/1.1 200") + assert got.endswith(b"PONG:PING") + finally: + far.close() + + +def test_the_workers_own_gate_refuses_our_network(): + # Every other gate here is built with a test rule; this is the one `proxy_url()` starts. + internal = _Listener() + try: + port = int(egress.proxy_url().rsplit(":", 1)[1]) + with socket.create_connection(("127.0.0.1", port), timeout=5) as s: + s.sendall(f"CONNECT {INTERNAL}:{internal.port} HTTP/1.1\r\n\r\n".encode()) + assert s.recv(4096).startswith(b"HTTP/1.1 403") + assert internal.hits == 0 + finally: + internal.close() + + +def _clip_hits(tmp_path: Path, segment: str, proxy: str) -> int: + """How many connections the internal listener takes while a clip (`download_ranges`, which + yt-dlp hands to ffmpeg even for an HLS format) is cut from a playlist whose one segment is + `segment` — formatted with the listener's address.""" + internal = _Listener() + playlist = ( + "#EXTM3U\n#EXT-X-VERSION:3\n#EXT-X-TARGETDURATION:2\n#EXTINF:2.0,\n" + + segment.format(host=INTERNAL, port=internal.port) + + "\n#EXT-X-ENDLIST\n" + ).encode() + public = _public_server( + {"/p.m3u8": (200, {"Content-Type": "application/vnd.apple.mpegurl"}, playlist)} + ) + try: + opts = _guarded_opts(tmp_path, proxy, download_ranges=download_range_func([], [(0, 1)])) + with contextlib.suppress(yt_dlp.utils.DownloadError), egress.open_ydl(opts) as ydl: + ydl.extract_info(f"http://{PUBLIC}:{public.server_address[1]}/p.m3u8", download=True) + return internal.hits + finally: + public.shutdown() + internal.close() + + +def test_a_clip_cut_by_ffmpeg_reaches_a_segment_only_through_the_gate(tmp_path, gate): + # Without the gate, ffmpeg fetches a hostile playlist's segment from inside our network — the + # control that shows this path is real. Through the gate it is never dialled. + segment = "http://{host}:{port}/s.ts" + assert _clip_hits(tmp_path / "bare", segment, proxy="") >= 1 + assert _clip_hits(tmp_path / "gated", segment, proxy=gate.url) == 0 + + +def test_ffmpeg_itself_refuses_a_raw_tcp_segment(tmp_path): + # A boundary, not our code: the HTTP proxy cannot see a tcp:// segment, and measured, ffmpeg's + # HLS reader does not open one from an http playlist. This fails if an ffmpeg upgrade starts to. + assert _clip_hits(tmp_path, "tcp://{host}:{port}/", proxy="") == 0 + + +# --- the wiring ------------------------------------------------------------------------------------ + + +def test_only_youtube_downloads_skip_the_gate(): + assert adapters.SourceAdapter.egress_guarded is True + assert adapters.resolve_adapter("url", "https://www.facebook.com/x").egress_guarded is True + assert adapters.resolve_adapter("url", "https://vimeo.com/1").egress_guarded is True + assert adapters.resolve_adapter("youtube", "dQw4w9WgXcQ").egress_guarded is False + + +def test_a_guarded_download_goes_through_the_gate_whatever_its_adapter_adds(): + opts = formats.build_ydl_opts( + {}, "/tmp/x", lambda d: None, extra={"proxy": "", "hls_prefer_native": False}, egress=True + ) + assert opts["proxy"] == egress.proxy_url() + assert opts["hls_prefer_native"] is True + assert opts[egress.GUARDED] is True + assert "proxy" not in formats.build_ydl_opts({}, "/tmp/x", lambda d: None, egress=False) + + +def test_every_yt_dlp_this_app_opens_goes_through_open_ydl(): + app = Path(egress.__file__).resolve().parents[1] + offenders = [ + f"{p.relative_to(app)}:{n}" + for p in sorted(app.rglob("*.py")) + if p.name != "egress.py" + for n, line in enumerate(p.read_text(encoding="utf-8").splitlines(), 1) + if "YoutubeDL(" in line + ] + assert offenders == [] diff --git a/backend/tests/test_facebook_impersonation.py b/backend/tests/test_facebook_impersonation.py index fc5c7840..f1a0971b 100644 --- a/backend/tests/test_facebook_impersonation.py +++ b/backend/tests/test_facebook_impersonation.py @@ -18,7 +18,7 @@ import yt_dlp from yt_dlp.networking.exceptions import TransportError from yt_dlp.networking.impersonate import ImpersonateTarget -from app.downloads import adapters, pipeline +from app.downloads import adapters, egress, pipeline from app.downloads import errors as dlerrors _URL = "https://www.facebook.com/reel/806379462330494/?fs=e&fs=e" @@ -73,6 +73,9 @@ def ydl(monkeypatch): def __enter__(self): return self + def add_post_processor(self, pp, when="post_process"): + pass # the egress format filter; nothing here to filter + def __exit__(self, *exc): return False @@ -157,6 +160,16 @@ def test_a_cookie_group_post_downloads_the_posts_own_video(ydl, tmp_path): # Never the post link: that is where the feed video comes from. assert web["extracted"] == [watch, watch] assert [o.get("impersonate") for o in seen[1:]] == [None, _CHROME] + # The post page read and both attempts all go through the egress gate. + assert all(o.get(egress.GUARDED) and o["proxy"] == egress.proxy_url() for o in seen), seen + + +def test_a_youtube_download_skips_the_gate(ydl, tmp_path): + seen, outcomes, _ = ydl + outcomes += [{"id": "dQw4w9WgXcQ"}] + yt = adapters.resolve_adapter("youtube", "dQw4w9WgXcQ") + pipeline._extract_with_fallback(1, 1, "https://youtu.be/dQw4w9WgXcQ", {}, tmp_path, yt) + assert seen and not any(o.get(egress.GUARDED) or o.get("proxy") for o in seen) def test_a_group_post_without_a_single_video_fails_saying_so(ydl, tmp_path): diff --git a/backend/tests/test_facebook_titles.py b/backend/tests/test_facebook_titles.py index 25372183..4bebe1c6 100644 --- a/backend/tests/test_facebook_titles.py +++ b/backend/tests/test_facebook_titles.py @@ -10,7 +10,7 @@ from sqlalchemy import select, update from app.config import settings from app.db import SessionLocal -from app.downloads import adapters, backfill, pipeline +from app.downloads import adapters, backfill, egress, pipeline from app.downloads.models import DownloadJob, MediaAsset _URL = "https://www.facebook.com/reel/1860682468422402/" @@ -36,6 +36,9 @@ def _fake_ydl(monkeypatch, outcome): def __enter__(self): return self + def add_post_processor(self, pp, when="post_process"): + pass # the egress format filter; nothing here to filter + def __exit__(self, *exc): return False @@ -43,7 +46,7 @@ def _fake_ydl(monkeypatch, outcome): seen.append({**self.opts, "url": url}) return outcome - monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", FakeYDL) + monkeypatch.setattr("yt_dlp.YoutubeDL", FakeYDL) return seen @@ -58,6 +61,7 @@ def test_a_placeholder_with_nothing_behind_it_takes_the_signed_out_pages_text(mo assert len(seen) == 1 assert seen[0]["skip_download"] is True assert not seen[0].get("cookiefile") + assert seen[0][egress.GUARDED] is True and seen[0]["proxy"] == egress.proxy_url() def test_a_real_title_reads_nothing_more(monkeypatch): diff --git a/backend/tests/test_failure_lines.py b/backend/tests/test_failure_lines.py index e5122140..de021f3a 100644 --- a/backend/tests/test_failure_lines.py +++ b/backend/tests/test_failure_lines.py @@ -359,6 +359,9 @@ def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkey def __enter__(self): return self + def add_post_processor(self, pp, when="post_process"): + pass # the egress format filter; nothing here to filter + def __exit__(self, *exc): return False @@ -367,7 +370,7 @@ def test_the_metadata_backfill_names_the_asset_and_not_the_download(sink, monkey f"ERROR: [youtube] dQw4w9WgXcQ: Video unavailable ({target})" ) - monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing) + monkeypatch.setattr("yt_dlp.YoutubeDL", _Refusing) assert backfill.extract_meta(url, adapters.YouTubeAdapter(), 11) is None lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]] @@ -386,13 +389,16 @@ def test_the_metadata_backfill_takes_out_a_generic_name_whatever_it_holds(sink, def __enter__(self): return self + def add_post_processor(self, pp, when="post_process"): + pass # the egress format filter; nothing here to filter + def __exit__(self, *exc): return False def extract_info(self, target, download): raise yt_dlp.utils.DownloadError("ERROR: [generic] Our Trip: Day 1: Unable to download") - monkeypatch.setattr(backfill.yt_dlp, "YoutubeDL", _Refusing) + monkeypatch.setattr("yt_dlp.YoutubeDL", _Refusing) assert backfill.extract_meta(url, adapters.SourceAdapter(), 11) is None lines = [o for o in sink.objects() if "meta extract failed" in o["msg"]] @@ -449,6 +455,9 @@ def test_a_retried_attempt_names_the_host_and_not_the_video(sink, monkeypatch, t def __enter__(self): return self + def add_post_processor(self, pp, when="post_process"): + pass # the egress format filter; nothing here to filter + def __exit__(self, *exc): return False @@ -463,6 +472,7 @@ def test_a_retried_attempt_names_the_host_and_not_the_video(sink, monkeypatch, t resolve_url=lambda target, opts: target, ydl_opts_extra=lambda clients: {}, retriable=lambda text: True, + egress_guarded=False, ) monkeypatch.setattr(pipeline.yt_dlp, "YoutubeDL", _YDL) monkeypatch.setattr(pipeline, "_touch_job", lambda job_id: None) diff --git a/backend/tests/test_formats.py b/backend/tests/test_formats.py index a0369c5d..d6ef2c61 100644 --- a/backend/tests/test_formats.py +++ b/backend/tests/test_formats.py @@ -15,16 +15,20 @@ from app.downloads import formats def _opts(spec): - return formats.build_ydl_opts(spec, "/tmp/%(id)s.%(ext)s", lambda d: None) + return formats.build_ydl_opts(spec, "/tmp/%(id)s.%(ext)s", lambda d: None, egress=False) def test_adapter_extra_opts_are_merged_in(): # The SourceAdapter contributes source-specific opts (e.g. YouTube's extractor_args) via `extra`. extra = {"extractor_args": {"youtube": {"player_client": ["web"]}}} - opts = formats.build_ydl_opts({}, "/tmp/%(id)s.%(ext)s", lambda d: None, extra=extra) + opts = formats.build_ydl_opts( + {}, "/tmp/%(id)s.%(ext)s", lambda d: None, extra=extra, egress=False + ) assert opts["extractor_args"]["youtube"]["player_client"] == ["web"] # No extra → no extractor_args (a generic source contributes nothing). - assert "extractor_args" not in formats.build_ydl_opts({}, "/tmp/x", lambda d: None) + assert "extractor_args" not in formats.build_ydl_opts( + {}, "/tmp/x", lambda d: None, egress=False + ) def test_mp4_default_selects_avc1_with_progressive_fallback(): @@ -113,7 +117,7 @@ def test_a_playlist_url_can_only_yield_one_item(): def test_the_remaining_quota_becomes_a_hard_size_ceiling(): - opts = formats.build_ydl_opts({}, "/tmp/x", lambda d: None, max_bytes=5_000_000) + opts = formats.build_ydl_opts({}, "/tmp/x", lambda d: None, max_bytes=5_000_000, egress=False) assert opts["max_filesize"] == 5_000_000 @@ -122,7 +126,7 @@ def test_no_budget_means_no_size_filter(): # enqueue, not an impossible yt-dlp filter to express here. for budget in (None, 0, -1): assert "max_filesize" not in formats.build_ydl_opts( - {}, "/tmp/x", lambda d: None, max_bytes=budget + {}, "/tmp/x", lambda d: None, max_bytes=budget, egress=False ) @@ -130,7 +134,12 @@ def test_a_source_adapter_cannot_raise_the_size_ceiling(): # The adapter merge happens after max_filesize is set, so this pins that no adapter option can # widen a storage cap — the mirror of the cookiefile rule, in the opposite direction. opts = formats.build_ydl_opts( - {}, "/tmp/x", lambda d: None, extra={"max_filesize": 999_999_999}, max_bytes=5_000_000 + {}, + "/tmp/x", + lambda d: None, + extra={"max_filesize": 999_999_999}, + max_bytes=5_000_000, + egress=False, ) assert opts["max_filesize"] == 5_000_000 @@ -197,7 +206,7 @@ def test_yt_dlp_refuses_a_file_over_the_cap_without_raising_and_the_watch_sees_i for cap, refused in ((1_000, True), (10_000_000, False)): out = tmp_path / f"out-{cap}" opts = formats.build_ydl_opts( - {}, str(out / "%(id)s.%(ext)s"), lambda d: None, max_bytes=cap + {}, str(out / "%(id)s.%(ext)s"), lambda d: None, max_bytes=cap, egress=False ) with formats.watching_size_cap() as over, yt_dlp.YoutubeDL(opts) as ydl: ydl.extract_info(url, download=True) diff --git a/backend/tests/test_migration_0110.py b/backend/tests/test_migration_0110.py index ba3677b8..82fe996a 100644 --- a/backend/tests/test_migration_0110.py +++ b/backend/tests/test_migration_0110.py @@ -45,6 +45,9 @@ def test_an_interval_override_moves_into_sysconfig_and_back(db): } assert conn.execute(text("SELECT to_regclass('scheduler_settings')")).scalar() is None assert scheduler.load_intervals(db)["rss_poll"] == 25 + # The way down rebuilds an index CONCURRENTLY (0117), which waits for every open transaction — + # the read above left this session's open. + db.rollback() command.downgrade(cfg, "0109_job_runs") try: diff --git a/backend/tests/test_migration_0119.py b/backend/tests/test_migration_0119.py new file mode 100644 index 00000000..6df41ccf --- /dev/null +++ b/backend/tests/test_migration_0119.py @@ -0,0 +1,70 @@ +"""Migration 0119's data step, DB-lane: the stored `transcode_hw` facet follows the rule that it says +whether the GPU does the encode, and a 10-bit decodable item is probed again so its HDR flag decides. + +The schema half is checked by test_migrations.py; a facet value left stale is invisible there. +""" + +from alembic import command +from alembic.config import Config +from sqlalchemy import text + +from app.db import engine + + +def test_the_stored_facet_is_rewritten_by_the_gpu_encode_rule(db): + cfg = Config("alembic.ini") + command.downgrade(cfg, "0118_videos_shuffle_key") + try: + with engine.begin() as conn: + lib = conn.execute( + text( + "INSERT INTO plex_libraries (plex_key, title, kind, enabled)" + " VALUES ('m1', 'Movies', 'movie', true) RETURNING id" + ) + ).scalar_one() + + def item(key, playable, codec, hw, depth=None, probed=False): + conn.execute( + text( + "INSERT INTO plex_items (rating_key, library_id, kind, title, playable," + " codec_video, transcode_hw, video_bit_depth, media_probed_at)" + " VALUES (:k, :l, 'movie', :k, :p, :c, :hw, :d," + " CASE WHEN :pr THEN now() END)" + ), + { + "k": key, + "l": lib, + "p": playable, + "c": codec, + "hw": hw, + "d": depth, + "pr": probed, + }, + ) + + item("vc1", "transcode", "VC1", True, 8, probed=True) # the old list said decodable + item("xvid", "transcode", "mpeg4", False) + item("divx", "transcode", "msmpeg4v3", False) + item("av1", "transcode", "av1", False) + item("h8", "transcode", "hevc", True, 8, probed=True) + item("h10", "transcode", "hevc", False, 10, probed=True) # SDR or HDR: unknown here + item("remux", "remux", "h264", None) + finally: + command.upgrade(cfg, "head") + + with engine.connect() as conn: + rows = { + k: (hw, probed) + for k, hw, probed in conn.execute( + text("SELECT rating_key, transcode_hw, media_probed_at IS NOT NULL FROM plex_items") + ).tuples() + } + assert rows == { + "vc1": (True, True), + "xvid": (True, False), + "divx": (True, False), + "av1": (False, False), + "h8": (True, True), + "h10": (False, False), # re-queued: the probe reads its HDR flag again + "remux": (None, False), + } diff --git a/backend/tests/test_plex_playback_class.py b/backend/tests/test_plex_playback_class.py index 39b36f0d..14a158ea 100644 --- a/backend/tests/test_plex_playback_class.py +++ b/backend/tests/test_plex_playback_class.py @@ -32,15 +32,25 @@ class TestTranscodeHwCapable: assert sync.transcode_hw_capable("transcode", "hevc") is True assert sync.transcode_hw_capable("transcode", "vp9", bit_depth=8) is True - def test_non_decodable_codec_is_sw(self): - # AV1 / MPEG-4 ASP (XviD) can't be HW-decoded on Gen 9.5 → software regardless of bit depth. + def test_a_codec_neither_path_knows_is_sw(self): assert sync.transcode_hw_capable("transcode", "av1") is False - assert sync.transcode_hw_capable("transcode", "mpeg4", bit_depth=8) is False + assert sync.transcode_hw_capable("transcode", "av1", bit_depth=8) is False - def test_10bit_or_hdr_hevc_is_sw(self): - # A HW-decodable codec that is 10-bit or HDR still breaks the VAAPI nv12 pipeline → software. + def test_a_cpu_decoded_8bit_codec_is_encoded_on_the_gpu(self): + # VC-1 (the iGPU has no VC-1 Advanced decode), XviD, DivX: the CPU decodes, the GPU encodes. + for codec in ("vc1", "mpeg4", "msmpeg4v3"): + assert sync.transcode_hw_capable("transcode", codec) is True + assert sync.transcode_hw_capable("transcode", "mpeg4", bit_depth=10) is False + + def test_10bit_sdr_hevc_is_sw(self): + # 10-bit SDR: an exact downconvert on the CPU (libx264), never the HDR tone curve. assert sync.transcode_hw_capable("transcode", "hevc", bit_depth=10) is False - assert sync.transcode_hw_capable("transcode", "hevc", bit_depth=8, hdr=True) is False + + def test_hdr_hevc_is_encoded_on_the_gpu(self): + # HDR goes through the OpenCL tonemap between VAAPI decode and encode (the prod image ships the + # runtime); the facet said software for it while every such session ran on the GPU. + assert sync.transcode_hw_capable("transcode", "hevc", bit_depth=10, hdr=True) is True + assert sync.transcode_hw_capable("transcode", "hevc", bit_depth=8, hdr=True) is True class TestVideoStreamProps: @@ -87,6 +97,9 @@ class TestSyncDerivation: row2 = PlexItem(rating_key="y", kind="movie") sync._apply_item(row2, lib_id=1, kind="movie", meta=self._meta("av1")) assert row2.transcode_hw is False # av1 not HW-decodable + row3 = PlexItem(rating_key="z", kind="movie") + sync._apply_item(row3, lib_id=1, kind="movie", meta=self._meta("vc1")) + assert row3.transcode_hw is True # CPU decode, GPU encode def test_probed_value_survives_resync_when_codec_unchanged(self): row = PlexItem(rating_key="x", kind="movie") @@ -124,21 +137,24 @@ class TestSyncDerivation: assert row.video_bit_depth is None and row.media_probed_at is None # re-probe scheduled -def test_migration_codec_set_matches_hw_decodable_codecs(): - # Migration 0077 hardcodes the HW-decodable codec set in raw SQL and its comment mandates it stay - # identical to plex.sync.HW_DECODABLE_CODECS. A comment isn't enforcement (house rule) — pin the - # equality so adding/removing a codec from the frozenset without editing the migration fails here - # (else already-migrated prod rows keep a wrong transcode_hw the runtime derivation wouldn't catch). +def test_migration_codec_sets_match_the_sync_sets(): + # The newest migration that rewrites `transcode_hw` hardcodes both codec sets in raw SQL. Pin them + # to plex.sync, so a codec added to or removed from a frozenset without a migration that rewrites + # the stored rows fails here (else prod rows keep a facet the runtime choice disagrees with). mig = ( Path(__file__).resolve().parent.parent / "alembic" / "versions" - / "0077_plex_playback_class.py" + / "0119_plex_transcode_gpu.py" ) - m = re.search(r"lower\(codec_video\)\s+IN\s+\(([^)]*)\)", mig.read_text()) - assert m, "could not find the codec IN(...) list in migration 0077" - codecs = {c.strip().strip("'\"").lower() for c in m.group(1).split(",")} - assert codecs == {c.lower() for c in sync.HW_DECODABLE_CODECS} + src = mig.read_text() + sets = {} + for name in ("HW_DECODABLE_CODECS", "CPU_DECODE_CODECS"): + m = re.search(name + r"\s*=\s*\(([^)]*)\)", src) + assert m, f"could not find {name} in migration 0119" + sets[name] = {c.strip().strip("'\"").lower() for c in m.group(1).split(",") if c.strip()} + assert sets["HW_DECODABLE_CODECS"] == {c.lower() for c in sync.HW_DECODABLE_CODECS} + assert sets["CPU_DECODE_CODECS"] == {c.lower() for c in sync.CPU_DECODE_CODECS} class _FakePlex: @@ -182,22 +198,42 @@ def test_probe_media_batch_refines_hw_to_sw_for_10bit(db): codec_video="mpeg4", transcode_hw=False, ) - db.add_all([hevc8, hevc10, xvid]) + hdr10 = PlexItem( + rating_key="hdr", + library_id=lib.id, + kind="movie", + title="hdr", + playable="transcode", + codec_video="hevc", + transcode_hw=True, + ) + db.add_all([hevc8, hevc10, xvid, hdr10]) db.commit() plex = _FakePlex( { "h8": {"Media": [{"Part": [{"Stream": [{"streamType": 1, "bitDepth": 8}]}]}]}, "h10": {"Media": [{"Part": [{"Stream": [{"streamType": 1, "bitDepth": 10}]}]}]}, + "hdr": { + "Media": [ + { + "Part": [ + {"Stream": [{"streamType": 1, "bitDepth": 10, "colorTrc": "smpte2084"}]} + ] + } + ] + }, } ) probed = sync._probe_media_batch(db, plex) db.refresh(hevc8) db.refresh(hevc10) db.refresh(xvid) - # Only the two HW-decodable HEVC items are probed; the XviD is SW-by-codec and never fetched. - assert probed == 2 + db.refresh(hdr10) + # Only the HW-decodable HEVC items are probed; the XviD's class follows from its codec alone. + assert probed == 3 assert hevc8.video_bit_depth == 8 and hevc8.transcode_hw is True and hevc8.media_probed_at - assert hevc10.video_bit_depth == 10 and hevc10.transcode_hw is False # downgraded to software + assert hevc10.video_bit_depth == 10 and hevc10.transcode_hw is False # 10-bit SDR: software + assert hdr10.video_bit_depth == 10 and hdr10.transcode_hw is True # HDR: the GPU tonemap assert xvid.media_probed_at is None # not in the work-list # A second run has nothing left to do (watermarks set). assert sync._probe_media_batch(db, plex) == 0 diff --git a/backend/tests/test_plex_stream.py b/backend/tests/test_plex_stream.py index 379d02e1..1b8e5901 100644 --- a/backend/tests/test_plex_stream.py +++ b/backend/tests/test_plex_stream.py @@ -284,6 +284,24 @@ class TestTranscodeCommand: ) assert cmd[cmd.index("-vf") + 1] == "scale_vaapi=format=nv12" # not a software scale + def test_cpu_decode_uploads_the_frames_to_the_gpu_encoder(self): + # A source the GPU cannot decode (VC-1 Advanced on Gen 9.5): the CPU decodes, the frames go to + # nv12 and up to the GPU, and h264_vaapi encodes. No -hwaccel: asking the GPU to decode is what + # failed. + cmd = stream._ffmpeg_cmd( + self.SRC, 0.0, self.OUT, None, transcode=True, hw=True, cpu_decode=True + ) + assert "-hwaccel" not in cmd + assert cmd[cmd.index("-init_hw_device") + 1] == f"vaapi=va:{stream._VAAPI_DEVICE}" + assert cmd[cmd.index("-filter_hw_device") + 1] == "va" + assert cmd.index("-init_hw_device") < cmd.index("-i") + assert cmd[cmd.index("-vf") + 1] == "format=nv12,hwupload" + assert _venc(cmd) == "h264_vaapi" + assert "libx264" not in cmd + assert ( + cmd[cmd.index("-force_key_frames") + 1] == f"expr:gte(t,n_forced*{stream._SEG_SECONDS})" + ) + def test_hdr_tonemap_uses_the_opencl_graph(self): # The HDR/10-bit HW path: VAAPI decode → OpenCL tonemap → VAAPI encode. Gen 9.5 can't tonemap in # the VAAPI VPP, so this is the only HW route for a 10-bit/HDR source. @@ -365,19 +383,48 @@ class TestWantHw: is True ) - def test_auto_falls_back_to_software_for_a_non_decodable_source(self, monkeypatch): - # AV1 / MPEG-4 ASP (XviD) aren't decodable on Gen 9.5 → the GPU can't feed its own encoder. + def test_auto_falls_back_to_software_for_a_codec_neither_path_knows(self, monkeypatch): + # AV1 is neither GPU-decodable on Gen 9.5 nor on the CPU-decode list (it can be 10-bit HDR). self._policy(monkeypatch, "auto") - self._hwsafe(monkeypatch, True) # decodability gate must fire BEFORE the probe + self._hwsafe(monkeypatch, True) # the codec gate must fire BEFORE the probe monkeypatch.setattr(stream, "_render_node_present", lambda: True) assert ( stream._want_hw(None, _FakeItem(codec_video="av1"), transcode=True, src=self.SRC) is False ) - assert ( - stream._want_hw(None, _FakeItem(codec_video="mpeg4"), transcode=True, src=self.SRC) - is False - ) + + @pytest.mark.parametrize("codec", ["vc1", "mpeg4", "msmpeg4v3"]) + def test_auto_cpu_decodes_what_the_gpu_cannot_and_still_encodes_on_it(self, monkeypatch, codec): + # Measured on prod (UHD 630, iHD 25.2.3): "No support for codec vc1 profile 3", so every VC-1 + # HW session died and fell back to libx264 (~19 s to start). Decoding on the CPU and encoding on + # the GPU produced three segments in ~5 s (VC-1 1080p) and ~1 s (XviD, DivX SD). + self._policy(monkeypatch, "auto") + self._kind(monkeypatch, stream._VK_HWSAFE) + self._opencl(monkeypatch, True) + monkeypatch.setattr(stream, "_render_node_present", lambda: True) + assert stream._select_encode( + None, _FakeItem(codec_video=codec), transcode=True, src=self.SRC + ) == (True, False, True) + + def test_auto_a_cpu_decoded_codec_that_is_not_8bit_sdr_stays_software(self, monkeypatch): + # The upload path converts to nv12 for the encoder; only an 8-bit SDR picture is known to come + # through it unchanged. + self._policy(monkeypatch, "auto") + self._kind(monkeypatch, stream._VK_SDR10) + monkeypatch.setattr(stream, "_render_node_present", lambda: True) + assert stream._select_encode( + None, _FakeItem(codec_video="mpeg4"), transcode=True, src=self.SRC + ) == (False, False, False) + + def test_on_cpu_decodes_a_codec_the_gpu_cannot(self, monkeypatch): + # "on" trusts the GPU, but asking it to DECODE a codec it is known not to decode only buys the + # fallback's wasted second; the upload path is the GPU route for such a source. + self._policy(monkeypatch, "on") + self._opencl(monkeypatch, False) + monkeypatch.setattr(stream, "_render_node_present", lambda: True) + assert stream._select_encode( + None, _FakeItem(codec_video="vc1"), transcode=True, src=self.SRC + ) == (True, False, True) def test_auto_hdr_source_falls_back_to_software_without_opencl(self, monkeypatch): # A decodable HEVC that is true-HDR (PQ/HLG) can't use scale_vaapi=format=nv12 (Gen 9.5), and with @@ -388,7 +435,7 @@ class TestWantHw: monkeypatch.setattr(stream, "_render_node_present", lambda: True) assert stream._select_encode( None, _FakeItem(codec_video="hevc"), transcode=True, src=self.SRC - ) == (False, False) + ) == (False, False, False) def test_auto_hdr_source_uses_hw_tonemap_with_opencl(self, monkeypatch): # With the OpenCL runtime present, a true-HDR HEVC tone-maps on the GPU (VAAPI decode → OpenCL @@ -399,7 +446,7 @@ class TestWantHw: monkeypatch.setattr(stream, "_render_node_present", lambda: True) assert stream._select_encode( None, _FakeItem(codec_video="hevc"), transcode=True, src=self.SRC - ) == (True, True) + ) == (True, True, False) def test_auto_10bit_SDR_source_uses_software_not_tonemap(self, monkeypatch): # F1 regression: a 10-bit *SDR* (bt709) source is NOT HDR, so it must NOT get the hable HDR tone @@ -411,7 +458,7 @@ class TestWantHw: monkeypatch.setattr(stream, "_render_node_present", lambda: True) assert stream._select_encode( None, _FakeItem(codec_video="hevc"), transcode=True, src=self.SRC - ) == (False, False) + ) == (False, False, False) def test_auto_8bit_source_uses_plain_hw_not_tonemap(self, monkeypatch): # An 8-bit SDR source takes the plain full-GPU scale path — never the tonemap graph — even where @@ -422,7 +469,7 @@ class TestWantHw: monkeypatch.setattr(stream, "_render_node_present", lambda: True) assert stream._select_encode( None, _FakeItem(codec_video="hevc"), transcode=True, src=self.SRC - ) == (True, False) + ) == (True, False, False) def test_off_forces_software_even_for_a_decodable_source(self, monkeypatch): self._policy(monkeypatch, "off") @@ -460,7 +507,7 @@ class TestWantHw: monkeypatch.setattr(stream, "_render_node_present", lambda: True) assert stream._select_encode( None, _FakeItem(codec_video="hevc"), transcode=True, src=self.SRC - ) == (True, True) + ) == (True, True, False) def test_on_10bit_SDR_uses_plain_hw_not_tonemap(self, monkeypatch): # F1 regression under "on": a 10-bit SDR source is NOT HDR, so even with OpenCL present "on" must @@ -471,7 +518,7 @@ class TestWantHw: monkeypatch.setattr(stream, "_render_node_present", lambda: True) assert stream._select_encode( None, _FakeItem(codec_video="hevc"), transcode=True, src=self.SRC - ) == (True, False) + ) == (True, False, False) def test_on_without_opencl_is_plain_hw(self, monkeypatch): # "on" with no OpenCL runtime → plain HW, tonemap=False, regardless of source class (the runtime @@ -483,7 +530,7 @@ class TestWantHw: monkeypatch.setattr(stream, "_render_node_present", lambda: True) assert stream._select_encode( None, _FakeItem(codec_video="hevc"), transcode=True, src=self.SRC - ) == (True, False) + ) == (True, False, False) class TestColdStartWarmup: @@ -620,6 +667,30 @@ class TestTranscodeStartSession: assert cmd[cmd.index("-vf") + 1] == "scale_vaapi=format=nv12" # no SW cap assert kwargs.get("preexec_fn") is None # GPU encode is cheap — never niced + def test_a_cpu_decode_session_encodes_on_the_gpu_and_falls_back_to_software( + self, capturing_env, monkeypatch + ): + monkeypatch.setattr( + stream, "_select_encode", lambda db, item, transcode, src: (True, False, True) + ) + seen = {"n": 0} + + def flaky_wait(path, timeout=20.0, proc=None): + seen["n"] += 1 + return seen["n"] > 1 # the upload session yields nothing; the software respawn does + + monkeypatch.setattr(stream, "wait_for", flaky_wait) + s = stream.start_session( + db=None, user_id=7, item=_FakeItem(codec_video="vc1"), start_s=0.0, transcode=True + ) + (up_cmd, *_), up_kwargs = _CapturingProc.calls[0] + assert "format=nv12,hwupload" in up_cmd and _venc(up_cmd) == "h264_vaapi" + if os.name == "posix": # the decode is CPU work: niced like a software transcode + assert up_kwargs.get("preexec_fn") is stream._sw_child_preexec + (sw_cmd, *_), _ = _CapturingProc.calls[1] + assert _venc(sw_cmd) == "libx264" and "format=nv12,hwupload" not in sw_cmd + assert s is not None and s.hw is False and s.cpu_decode is False + def test_hdr_tonemap_transcode_threads_the_opencl_graph_and_flags_the_session( self, capturing_env, monkeypatch ): diff --git a/backend/tests/test_reader_service.py b/backend/tests/test_reader_service.py index 647c6825..7d54e648 100644 --- a/backend/tests/test_reader_service.py +++ b/backend/tests/test_reader_service.py @@ -252,7 +252,7 @@ def test_an_add_is_idempotent_and_lands_where_asked(db, make_user): db.commit() assert _ids(db, pl) == ["c", "b"] assert queries.item_counts(db, [pl.id]) == {pl.id: 2} - assert queries.durations(db, [pl.id]) == {pl.id: 120} + assert queries.durations(db, [pl.id]) == {pl.id: (120, 0)} assert queries.containing(db, [pl.id], "b") == {pl.id} @@ -294,7 +294,7 @@ def test_a_rename_and_a_delete(db, make_user): service.add_item(db, pl, "a", mark_dirty=False) service.rename_playlist(db, pl, "New") db.commit() - assert queries.playlist_totals(db, pl.id) == (1, 60, None) + assert queries.playlist_totals(db, pl.id) == (1, 60, 0, None) assert pl.name == "New" service.delete_playlist(db, pl) db.commit() diff --git a/backend/tests/test_shuffle_walk.py b/backend/tests/test_shuffle_walk.py new file mode 100644 index 00000000..bb19ed01 --- /dev/null +++ b/backend/tests/test_shuffle_walk.py @@ -0,0 +1,223 @@ +"""A seeded shuffle walks a stored random key instead of sorting the whole scope. DB lane. + +Every video carries a random number drawn once (`Video.shuffle_key`). A seed picks a point on that +ring; the feed's Shuffle sort pages along the ring from there and wraps, and the Home surprise reads +a window of it and shuffles that window by the seed (`discover.queries.yt_surprise`). Ordering the +scope by md5(id || seed) cost ~350 ms per call on prod and grew with every Shuffle page. +""" + +import base64 +import random +from contextlib import contextmanager +from datetime import datetime, timedelta, timezone + +from sqlalchemy import event, insert, text + +from app import feedquery +from app.catalog.models import Channel, Subscription, Video +from app.discover import queries +from app.discover.queries import _FEED_BASE + +NOW = datetime.now(timezone.utc) +KEYS = {"k05": 0.05, "k20": 0.20, "k40": 0.40, "k60": 0.60, "k80": 0.80, "k95": 0.95} + + +def _seed(db, make_user, keys=KEYS): + user = make_user() + db.add(Channel(id="UCring", title="Ring")) + db.flush() + db.add(Subscription(user_id=user.id, channel_id="UCring")) + db.add_all( + [ + Video( + id=vid, + channel_id="UCring", + title=vid, + duration_seconds=20 * 60, + published_at=NOW - timedelta(days=i + 1), + shuffle_key=key, + ) + for i, (vid, key) in enumerate(keys.items()) + ] + ) + db.commit() + return user + + +def _ring(seed, keys=KEYS): + """The ring order from the seed's point: the keys at or after it ascending, then the rest.""" + r = feedquery.shuffle_start(seed) + return sorted(keys, key=lambda v: (keys[v] < r, keys[v])) + + +def _seed_landing_between(lo, hi): + return next(s for s in range(1000) if lo < feedquery.shuffle_start(s) < hi) + + +def test_the_walk_starts_at_the_seeds_point_and_wraps(db, make_user): + user = _seed(db, make_user) + query, _ = feedquery.filtered_query(db, user, **_FEED_BASE) + seed = _seed_landing_between(0.45, 0.55) # starts at k60, wraps to k05 after k95 + got = [r.id for r in feedquery.shuffle_walk(db, query, seed, 6)] + assert got == ["k60", "k80", "k95", "k05", "k20", "k40"] + assert [r.id for r in feedquery.shuffle_walk(db, query, seed, 4)] == got[:4] + + +def test_the_feed_shuffle_pages_along_the_ring(db, make_user, as_user): + user = _seed(db, make_user) + client = as_user(user) + for seed in (_seed_landing_between(0.1, 0.15), _seed_landing_between(0.85, 0.9)): + walked, cursor = [], None + for _ in range(5): + url = f"/api/feed?sort=shuffle&seed={seed}&limit=2" + ( + f"&cursor={cursor}" if cursor else "" + ) + body = client.get(url).json() + walked += [it["id"] for it in body["items"]] + cursor = body["next_cursor"] + if not body["has_more"]: + break + assert walked == _ring(seed), seed + + +def test_a_new_seed_opens_the_shuffle_elsewhere(db, make_user, as_user): + user = _seed(db, make_user) + client = as_user(user) + first = { + seed: client.get(f"/api/feed?sort=shuffle&seed={seed}&limit=2").json()["items"][0]["id"] + for seed in (_seed_landing_between(0.1, 0.15), _seed_landing_between(0.65, 0.7)) + } + assert sorted(first.values()) == ["k20", "k80"] + + +@contextmanager +def _statements(db): + seen: list[str] = [] + + def grab(conn, cursor, statement, *args): + seen.append(statement) + + engine = db.get_bind() + event.listen(engine, "before_cursor_execute", grab) + try: + yield seen + finally: + event.remove(engine, "before_cursor_execute", grab) + + +def _draws_of_videos(seen): + return [s for s in seen if "FROM videos" in s and "LIMIT" in s] + + +def test_neither_draw_orders_the_scope_by_a_hash(db, make_user): + # Each draw reads the videos as an ascending range of the key (what ix_videos_shuffle_key serves), + # never ordered by a hash or by the ring's CASE (which no index can serve). + user = _seed(db, make_user) + with _statements(db) as surprise: + queries.yt_surprise(db, user, "any", 7, 3) + with _statements(db) as page: + feedquery.feed_page( + db, user, dict(_FEED_BASE), sort="shuffle", seed=7, limit=2, cursor=None, lang="en" + ) + for seen in (surprise, page): + draws = _draws_of_videos(seen) + assert draws + for s in draws: + assert "md5(" not in s.lower() + assert "ORDER BY videos.shuffle_key, videos.id" in s.replace("\n", " "), s + + +def test_the_walk_reads_the_ring_index_and_sorts_nothing(db, make_user): + # The text of the query is not the plan: each arc must be served by ix_videos_shuffle_key in key + # order (an index on another shape, or none, leaves a Sort of the whole scope). A scope of a few + # thousand videos, analysed, so the planner weighs it as it does a real one. + user = _seed(db, make_user) + rng = random.Random(7) + db.execute( + insert(Video), + [ + { + "id": f"bulk{i:05}", + "channel_id": "UCring", + "title": "b", + "duration_seconds": 600, + "published_at": NOW - timedelta(minutes=i), + "shuffle_key": rng.random(), + } + for i in range(4000) + ], + ) + db.commit() + db.execute(text("ANALYZE videos")) + query, _ = feedquery.filtered_query(db, user, **_FEED_BASE) + captured: list[tuple[str, object]] = [] + + def grab(conn, cursor, statement, params, *args): + if "FROM videos" in statement and "LIMIT" in statement: + captured.append((statement, params)) + + engine = db.get_bind() + event.listen(engine, "before_cursor_execute", grab) + try: + seed = _seed_landing_between(0.45, 0.55) + feedquery.shuffle_walk(db, query, seed, 6) # the first arc, from the seed's point + feedquery.shuffle_walk(db, query, seed, 6, (1, 0.3, "bulk00001")) # the wrap, resumed + finally: + event.remove(engine, "before_cursor_execute", grab) + assert len(captured) == 2 + conn = db.connection() + for statement, params in captured: + plan = "\n".join(r[0] for r in conn.exec_driver_sql("EXPLAIN " + statement, params)) + assert "ix_videos_shuffle_key" in plan, plan + assert "Sort Key: videos.shuffle_key" not in plan, plan + + +def test_the_surprise_reads_a_window_of_the_ring(db, make_user, monkeypatch): + user = _seed(db, make_user) + monkeypatch.setattr(queries, "SURPRISE_WINDOW", 3) + for seed in range(20): + drawn = {r["id"] for r in queries.yt_surprise(db, user, "any", seed, 3)} + assert drawn == set(_ring(seed)[:3]), seed + + +def test_a_wide_gap_on_the_ring_does_not_make_a_favourite(db, make_user): + # One video after an empty half of the ring, twenty packed into the other half. Taking the first + # video after the seed's point would deal the lonely one about half the time; the window is + # shuffled by the seed, so it leads about as often as any other. + keys = {"lonely": 0.5} | {f"p{i:02}": 0.51 + i * 0.02 for i in range(20)} + user = _seed(db, make_user, keys) + leads = [queries.yt_surprise(db, user, "any", s, 1)[0]["id"] for s in range(90)] + assert leads.count("lonely") < 15, leads.count("lonely") + + +def test_any_seed_lands_on_the_ring(): + # An int too large for a float overflowed the multiplication: a 500 for a hand-typed seed. + for seed in (10**400, -(10**400), 2**53 + 1, -7): + assert 0 <= feedquery.shuffle_start(seed) < 1, seed + + +def test_a_seed_out_of_range_is_refused_not_a_500(db, make_user, as_user): + client = as_user(_seed(db, make_user)) + huge = "9" * 400 + for url in ( + f"/api/feed?sort=shuffle&seed={huge}", + f"/api/discover?seed={huge}", + f"/api/discover/pick?seed={huge}", + ): + assert client.get(url).status_code == 422, url + + +def test_a_shuffle_cursor_past_a_float_is_a_bad_cursor(db, make_user, as_user): + client = as_user(_seed(db, make_user)) + for k in ("[Infinity, 0.5]", f"[0, {'9' * 400}]"): + raw = '{"k": %s, "id": "k20"}' % k + cursor = base64.urlsafe_b64encode(raw.encode()).decode() + assert client.get(f"/api/feed?sort=shuffle&seed=1&cursor={cursor}").status_code == 400, k + + +def test_a_new_video_gets_its_own_key(db, make_user): + _seed(db, make_user, {}) + db.add_all([Video(id=f"n{i}", channel_id="UCring", title="n") for i in range(3)]) + db.commit() + keys = [db.get(Video, f"n{i}").shuffle_key for i in range(3)] + assert len(set(keys)) == 3 and all(0 <= k < 1 for k in keys) diff --git a/backend/tests/test_surprise_affinity.py b/backend/tests/test_surprise_affinity.py new file mode 100644 index 00000000..ba828e1d --- /dev/null +++ b/backend/tests/test_surprise_affinity.py @@ -0,0 +1,349 @@ +"""Surprise me leans on what the user watches, and stays a surprise. + +The Home hero's surprise tier and the Plex Discover rail were uniform draws over the unseen. They are +now weighted draws (`app.draw`): a title's weight is 1 + its affinity — for Plex the genres of the +films and shows the user watched (recent ones weigh more) and the TMDB recommendations stored on them; +for YouTube the channels the user watches. One slot of three stays a blind draw, and with no watch +history the order is exactly the old one. DB lane. +""" + +import hashlib +from datetime import datetime, timedelta, timezone + +from sqlalchemy import select, text + +from app import draw +from app.catalog.models import Channel, Subscription, Video +from app.discover import queries +from app.plex import affinity, discover as plex_discover +from app.plex.models import PlexItem, PlexLibrary, PlexShow, PlexState +from app.plex.query import LibraryFilters +from app.reader.models import VideoState + +NOW = datetime.now(timezone.utc) +SEEDS = range(90) + + +# ── the draw itself ────────────────────────────────────────────────────────────────────────────── + + +def test_every_third_slot_is_blind_and_nothing_repeats(): + weighted = ["a", "b", "c", "d", "e", "f"] + blind = ["f", "e", "d", "c", "b", "a"] + # offset 0: slots 0 and 3 are blind + assert draw.blend(weighted, blind, 0, 6) == ["f", "a", "b", "e", "c", "d"] + # offset 1: slot 2 and 5 + assert draw.blend(weighted, blind, 1, 6) == ["a", "b", "f", "c", "d", "e"] + assert draw.blend(weighted, blind, 2, 3) == ["a", "f", "b"] + + +def test_a_short_side_is_topped_up_from_the_other(): + assert draw.blend(["a"], ["a", "b", "c"], 0, 3) == ["a", "b", "c"] + assert draw.blend(["a", "b", "c"], [], 0, 3) == ["a", "b", "c"] + + +def test_a_watch_weighs_by_its_age_an_undated_one_two_half_lives_and_a_future_one_now(): + # Both affinities weigh every watch through this one rule; an old import has no date, and a + # clock ahead of the server's must not weigh more than a watch made now. + assert draw.recency(NOW, NOW, 30) == 1.0 + assert draw.recency(NOW - timedelta(days=30), NOW, 30) == 0.5 + assert draw.recency(None, NOW, 30) == 0.25 + assert draw.recency(NOW + timedelta(days=5), NOW, 30) == 1.0 + + +def test_scores_are_scaled_to_the_strongest_and_nothing_scored_is_empty(): + assert draw.scaled({"a": 4.0, "b": 1.0}) == {"a": 1.0, "b": 0.25} + assert draw.scaled({}) == {} + + +# ── YouTube: the channels the user watches ─────────────────────────────────────────────────────── + + +def _md5_order(ids, seed): + return sorted(ids, key=lambda i: hashlib.md5(f"{i}{seed}".encode()).hexdigest()) + + +def _seed_yt(db, make_user, *, watched_in_fav: int): + user = make_user() + fav, other = Channel(id="UCfav", title="Fav"), Channel(id="UCother", title="Other") + db.add_all([fav, other]) + db.flush() + db.add_all( + [ + Subscription(user_id=user.id, channel_id="UCfav"), + Subscription(user_id=user.id, channel_id="UCother"), + ] + ) + + def video(vid, ch, days): + return Video( + id=vid, + channel_id=ch, + title=vid, + duration_seconds=20 * 60, + published_at=NOW - timedelta(days=days), + ) + + # One unseen video per channel, plus the user's watched history on the favourite. + db.add_all([video("fav_new", "UCfav", 1), video("other_new", "UCother", 1)]) + for i in range(watched_in_fav): + db.add(video(f"fav_seen{i}", "UCfav", 10 + i)) + db.flush() + for i in range(watched_in_fav): + db.add( + VideoState( + user_id=user.id, + video_id=f"fav_seen{i}", + status="watched", + watched_at=NOW - timedelta(days=i), + ) + ) + db.commit() + return user + + +def _first(rows): + return rows[0]["id"] if rows else None + + +def test_a_watched_channel_wins_most_draws_but_not_all(db, make_user): + user = _seed_yt(db, make_user, watched_in_fav=12) + leads = [_first(queries.yt_surprise(db, user, "any", s, 2)) for s in SEEDS] + fav = leads.count("fav_new") / len(leads) + # weight 1 + 3 against 1: 0.8 in a weighted slot, 0.5 in the blind one (a third of the seeds) + assert 0.6 < fav < 0.9, fav + + +def test_the_blind_slot_deals_the_plain_shuffle_however_strong_the_lean(db, make_user): + user = _seed_yt(db, make_user, watched_in_fav=12) + for s in range(0, 30, 3): # (0 + seed) % 3 == 0: the first slot is the blind one + got = _first(queries.yt_surprise(db, user, "any", s, 2)) + assert got == _md5_order(["fav_new", "other_new"], s)[0] + + +def test_with_no_history_the_youtube_order_is_the_old_shuffle(db, make_user): + user = _seed_yt(db, make_user, watched_in_fav=0) + for s in range(12): + got = [r["id"] for r in queries.yt_surprise(db, user, "any", s, 2)] + assert got == _md5_order(["fav_new", "other_new"], s) + + +def test_a_watched_video_is_never_drawn(db, make_user): + user = _seed_yt(db, make_user, watched_in_fav=12) + drawn = {r["id"] for s in SEEDS for r in queries.yt_surprise(db, user, "any", s, 5)} + assert drawn == {"fav_new", "other_new"} + + +# ── Plex: genres and recommendations of what the user watched ──────────────────────────────────── + + +def _seed_plex(db, make_user, *, history: bool): + user = make_user() + lib = PlexLibrary(plex_key="77", title="Movies", kind="movie", enabled=True) + db.add(lib) + db.flush() + tmdb_ids = iter(range(5000, 6000)) + + def movie(key, genres, recs=None): + m = PlexItem( + rating_key=key, + library_id=lib.id, + kind="movie", + title=key, + duration_s=100 * 60, + genres=genres, + guid=f"tmdb:{next(tmdb_ids)}", + recommendations=recs, + ) + db.add(m) + return m + + horror = movie("horror_new", ["Horror"]) + comedy = movie("comedy_new", ["Comedy"]) + tipped = movie("tipped_new", ["Western"]) + db.flush() + tip = [{"tmdb_id": int(tipped.guid.split(":")[1]), "media_type": "movie", "title": "t"}] + seen = [movie(f"horror_seen{i}", ["Horror"], tip if i == 0 else None) for i in range(3)] + db.flush() + if history: + for i, m in enumerate(seen): + db.add( + PlexState( + user_id=user.id, + item_id=m.id, + status="watched", + watched_at=NOW - timedelta(days=i), + ) + ) + db.commit() + from app import sysconfig + + sysconfig.set_value(db, "plex_enabled", True) + db.commit() + return user, horror, comedy, tipped + + +def test_the_genre_and_the_recommendation_of_what_was_watched_win_most_draws(db, make_user): + user, *_ = _seed_plex(db, make_user, history=True) + leads = [_first(queries.plex_surprise(db, user.id, "any", s, 3, lang="en")) for s in SEEDS] + # Weights: Horror 1 + 2 (its genre), the recommended Western 1 + 3, the Comedy nobody chose 1. In + # a weighted slot the Comedy leads 1 time in 8, in the blind third 1 in 3: about 0.19 against + # 0.81 for the other two together (a uniform draw: 1 in 3 each). It still leads now and then — + # a surprise, not a ranking. + comedy = leads.count("comedy_new") + assert comedy > 0 + assert leads.count("horror_new") > 1.5 * comedy and leads.count("tipped_new") > 1.5 * comedy + assert leads.count("horror_new") + leads.count("tipped_new") > 3 * comedy + + +def test_with_no_history_the_plex_order_is_the_old_shuffle(db, make_user): + user, *_ = _seed_plex(db, make_user, history=False) + keys = ["horror_new", "comedy_new", "tipped_new"] + [f"horror_seen{i}" for i in range(3)] + for s in range(12): + got = [c["id"] for c in queries.plex_surprise(db, user.id, "any", s, 6, lang="en")] + assert got == _md5_order(keys, s) + + +def test_a_watched_film_is_never_drawn(db, make_user): + user, *_ = _seed_plex(db, make_user, history=True) + drawn = { + c["id"] for s in SEEDS for c in queries.plex_surprise(db, user.id, "any", s, 6, lang="en") + } + assert drawn == {"horror_new", "comedy_new", "tipped_new"} + + +def test_the_plex_blind_slot_deals_the_plain_shuffle(db, make_user): + user, *_ = _seed_plex(db, make_user, history=True) + unseen = ["horror_new", "comedy_new", "tipped_new"] + for s in range(0, 30, 3): + got = _first(queries.plex_surprise(db, user.id, "any", s, 3, lang="en")) + assert got == _md5_order(unseen, s)[0] + + +def test_a_binged_show_counts_once_and_recent_watches_weigh_more(db, make_user): + user = make_user() + lib = PlexLibrary(plex_key="78", title="Shows", kind="show", enabled=True) + mlib = PlexLibrary(plex_key="79", title="Movies", kind="movie", enabled=True) + db.add_all([lib, mlib]) + db.flush() + show = PlexShow(rating_key="sitcom", library_id=lib.id, title="Sitcom", genres=["Comedy"]) + db.add(show) + db.flush() + eps = [ + PlexItem( + rating_key=f"ep{i}", + library_id=lib.id, + kind="episode", + title=f"e{i}", + show_id=show.id, + season_number=1, + episode_number=i, + ) + for i in range(10) + ] + films = [ + PlexItem( + rating_key=f"h{i}", library_id=mlib.id, kind="movie", title=f"h{i}", genres=["Horror"] + ) + for i in range(2) + ] + old = PlexItem( + rating_key="w0", library_id=mlib.id, kind="movie", title="w0", genres=["Western"] + ) + db.add_all([*eps, *films, old]) + db.flush() + for it in [*eps, *films]: + db.add(PlexState(user_id=user.id, item_id=it.id, status="watched", watched_at=NOW)) + db.add( + PlexState( + user_id=user.id, item_id=old.id, status="watched", watched_at=NOW - timedelta(days=720) + ) + ) + db.commit() + + genres = affinity.for_user(db, user.id).genres + # Ten episodes of one show are one title: two horror films outweigh it. + assert genres["Horror"] == 1.0 + assert genres["Comedy"] == 0.5 + # Four half-lives ago weighs a sixteenth of today. + assert abs(genres["Western"] - 0.5 / 16) < 0.01 + + +def test_the_discover_rail_leans_the_same_way(db, make_user): + # The rail draws with random(), not a seed; setseed makes this session's draws repeatable. + user, *_ = _seed_plex(db, make_user, history=True) + db.execute(text("SELECT setseed(0.42)")) + leads = [ + plex_discover.surprise(db, user, "movie", "en", 3, [], [], LibraryFilters())["items"][0][ + "id" + ] + for _ in SEEDS + ] + assert leads.count("comedy_new") > 0 + assert leads.count("horror_new") + leads.count("tipped_new") > 3 * leads.count("comedy_new") + + +# ── whose history: only the user's own ─────────────────────────────────────────────────────────── + + +def test_another_users_youtube_history_does_not_lean_my_draw(db, make_user): + watcher = _seed_yt(db, make_user, watched_in_fav=12) + me = make_user() + db.add_all( + [ + Subscription(user_id=me.id, channel_id="UCfav"), + Subscription(user_id=me.id, channel_id="UCother"), + ] + ) + db.commit() + assert queries._channel_affinity(db, watcher) + assert queries._channel_affinity(db, me) == {} + + +def test_another_users_plex_history_does_not_lean_my_draw(db, make_user): + _seed_plex(db, make_user, history=True) + me = make_user() + assert not affinity.for_user(db, me.id) + keys = ["horror_new", "comedy_new", "tipped_new"] + [f"horror_seen{i}" for i in range(3)] + for s in range(12): + got = [c["id"] for c in queries.plex_surprise(db, me.id, "any", s, 6, lang="en")] + assert got == _md5_order(keys, s) + + +# ── the rail: its blind slot, and a show's own recommendations ─────────────────────────────────── + + +def test_the_rails_third_slot_is_the_blind_draw(db, make_user, monkeypatch): + # The rail draws with random(), so the blend is watched rather than its odds: every leaning draw + # goes through it, with the blind order beside the weighted one and the third slot blind. + user, *_ = _seed_plex(db, make_user, history=True) + calls = [] + real = draw.blend + + def spy(weighted, blind, offset, n, key): + calls.append((offset, n, sorted(map(key, weighted)), sorted(map(key, blind)))) + return real(weighted, blind, offset, n, key=key) + + monkeypatch.setattr(draw, "blend", spy) + got = plex_discover.surprise(db, user, "movie", "en", 3, [], [], LibraryFilters())["items"] + assert calls and all(offset == 1 and n == 3 for offset, n, *_ in calls) + pool = sorted(["horror_new", "comedy_new", "tipped_new"]) + assert calls[0][2] == pool and calls[0][3] == pool + assert len(got) == 3 + + +def test_a_show_leans_on_the_shows_recommended_not_the_films(db, make_user): + lib = PlexLibrary(plex_key="80", title="Shows", kind="show", enabled=True) + db.add(lib) + db.flush() + tipped = PlexShow(rating_key="tv1", library_id=lib.id, title="t", guid="tmdb:1", genres=[]) + db.add(tipped) + db.commit() + shows = affinity.Affinity(show_recs={"tmdb:1": 1.0}) + films = affinity.Affinity(movie_recs={"tmdb:1": 1.0}) + + def weight(aff): + return db.execute(select(aff.weight(PlexShow)).where(PlexShow.id == tipped.id)).scalar_one() + + assert weight(shows) == 1 + affinity.REC_WEIGHT + assert weight(films) == 1 diff --git a/backend/tests/test_video_length.py b/backend/tests/test_video_length.py new file mode 100644 index 00000000..f288601e --- /dev/null +++ b/backend/tests/test_video_length.py @@ -0,0 +1,218 @@ +"""A YouTube length past a day is unknown, everywhere a length is used (`catalog.models.LENGTH_MAX_S`). + +Measured on prod (2026-11-06): two 24/7 streams' archives report 122,962,914 s and 34,762,459 s, and the +owner's Watch later total read "44,073 hr 55 min". Such a video now has no length on any API answer, the +totals leave it out (a playlist's counts it as `without_length`), and the duration filter and sort pass it over +like a video whose length is not fetched yet. An AST check holds who may still read the raw column. +DB lane (the AST check needs none). +""" + +import ast +from datetime import datetime, timezone +from pathlib import Path + +from sqlalchemy.dialects import postgresql + +from app.catalog.models import LENGTH_MAX_S, Channel, Subscription, Video, known_length +from app.reader.models import Playlist, PlaylistItem + +YEARS = 122_962_914 # the measured archive +HOUR = 3600 + + +def _seed(db, user): + db.add(Channel(id="UClen", title="Len")) + db.flush() + db.add(Subscription(user_id=user.id, channel_id="UClen", priority=0, hidden=False)) + rows = [ + ("short", 600, "none"), + ("long", 2 * HOUR, "none"), + ("day", LENGTH_MAX_S, "was_live"), # exactly a day still is a length + ("archive", YEARS, "was_live"), + ("onair", None, "live"), + ("unfetched", None, "none"), # not live, but left out of the sum: counted all the same + ] + for i, (vid, seconds, live) in enumerate(rows): + db.add( + Video( + id=vid, + channel_id="UClen", + published_at=datetime(2024, 1, i + 1, tzinfo=timezone.utc), + title=vid, + duration_seconds=seconds, + live_status=live, + is_short=False, + via_search=False, + via_explore=False, + ) + ) + pl = Playlist(user_id=user.id, name="Later") + db.add(pl) + db.flush() + for i, (vid, _s, _l) in enumerate(rows): + db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=i)) + db.commit() + return pl + + +KNOWN_SUM = 600 + 2 * HOUR + LENGTH_MAX_S + + +def test_the_rule(): + assert known_length(None) is None + assert known_length(LENGTH_MAX_S) == LENGTH_MAX_S + assert known_length(LENGTH_MAX_S + 1) is None + assert known_length(YEARS) is None + + +def test_the_sql_length_carries_its_limit_as_a_literal(): + # The duration sort indexes (migration 0117) are built on this expression with 86400 in it. A + # bound parameter matches no index once the driver reuses a prepared, generic plan, and the + # sort-index test cannot see that: it compiles with literal binds. + compiled = Video.length_seconds.expression.compile(dialect=postgresql.dialect()) + assert compiled.params == {} + assert f"> {LENGTH_MAX_S}" in str(compiled) + + +def test_a_years_long_archive_has_no_length_in_the_feed_or_a_playlist(db, make_user, as_user): + user = make_user() + pl = _seed(db, user) + client = as_user(user) + + feed = client.get("/api/feed?show=all&include_live=true&limit=50") + assert feed.status_code == 200 + by_id = {v["id"]: v["duration_seconds"] for v in feed.json()["items"]} + assert by_id["archive"] is None + assert by_id["day"] == LENGTH_MAX_S + + detail = client.get(f"/api/playlists/{pl.id}") + assert detail.status_code == 200 + items = {v["id"]: v["duration_seconds"] for v in detail.json()["items"]} + assert items["archive"] is None + + one = client.get("/api/videos/archive") + assert one.status_code == 200 + assert one.json()["duration_seconds"] is None + + +def test_an_off_catalog_archive_has_no_length_either(db, make_user, as_user, monkeypatch): + # A video not in the catalog is looked up on YouTube (videos.list); its length is judged the same. + from app.routes import feed as feed_routes + + class Yt: + def __init__(self, *_): + pass + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def get_videos(self, ids): + hours = {"offyears": YEARS // HOUR, "offshort": 1}[ids[0]] + return [ + {"snippet": {}, "statistics": {}, "contentDetails": {"duration": f"PT{hours}H"}} + ] + + monkeypatch.setattr(feed_routes, "YouTubeClient", Yt) + client = as_user(make_user()) + assert client.get("/api/videos/offyears").json()["duration_seconds"] is None + assert client.get("/api/videos/offshort").json()["duration_seconds"] == HOUR + + +def test_the_totals_leave_it_out_and_count_what_they_left_out(db, make_user, as_user): + user = make_user() + pl = _seed(db, user) + client = as_user(user) + + listed = client.get("/api/playlists") + assert listed.status_code == 200 + (summary,) = [p for p in listed.json() if p["id"] == pl.id] + assert summary["total_duration_seconds"] == KNOWN_SUM + assert summary["without_length"] == 3 # the archive, the one on air, the unfetched one + + renamed = client.patch(f"/api/playlists/{pl.id}", json={"name": "Later 2"}) + assert renamed.status_code == 200 + assert renamed.json()["total_duration_seconds"] == KNOWN_SUM + assert renamed.json()["without_length"] == 3 + + channels = client.get("/api/channels") + assert channels.status_code == 200 + (ch,) = channels.json() + assert ch["total_duration_seconds"] == KNOWN_SUM + # The channel table shows the sum alone (the count did not fit its column), so it is not sent. + assert "without_length" not in ch and "live_without_length" not in ch + + +def test_the_duration_filter_and_sort_pass_it_over(db, make_user, as_user): + user = make_user() + _seed(db, user) + client = as_user(user) + base = "/api/feed?show=all&include_live=true&limit=50" + + longer = client.get(f"{base}&min_duration={HOUR}") + assert longer.status_code == 200 + assert {v["id"] for v in longer.json()["items"]} == {"long", "day"} + + longest = client.get(f"{base}&sort=duration_desc") + assert longest.status_code == 200 + order = [v["id"] for v in longest.json()["items"]] + assert order[:3] == ["day", "long", "short"] + assert set(order[3:]) == {"archive", "onair", "unfetched"} + + +# Who may read `duration_seconds` itself, by (file, function): what each one asks is not "how long". +RAW_READERS = { + # The column is defined and turned into a length here. + ("catalog/models.py", "length_seconds"), + ("catalog/models.py", "_length_seconds_expression"), + # Ingest writes it; the live refresh asks whether YouTube has answered for an archive yet; the + # Shorts classification needs only "longer than a Short", which a years-long value proves. + ("sync/videos.py", "apply_video_details"), + ("sync/videos.py", "refresh_live"), + ("sync/videos.py", "run_shorts_classification"), + ("catalog/search.py", "classify_shorts"), + ("catalog/queries.py", "work_queue"), + # Not a video's column: the player's own report, and the watch history's looked-up lengths, which + # count a watched video as more than a glance (a watched archive is that too). + ("routes/_common.py", "kept_position"), + ("routes/plex/playback.py", "item_progress"), + ("reader/queries.py", "_counted_channels"), + ("reader/queries.py", "watched_channels"), + ("reader/queries.py", "watched_channel_videos"), +} + + +def _raw_reads() -> set[tuple[str, str]]: + root = Path(__file__).resolve().parents[1] / "app" + found: set[tuple[str, str]] = set() + for path in root.rglob("*.py"): + if "alembic" in path.parts: + continue + rel = path.relative_to(root).as_posix() + tree = ast.parse(path.read_text(encoding="utf-8")) + in_a_function: set[int] = set() + for fn in ast.walk(tree): + if not isinstance(fn, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + for node in ast.walk(fn): + if _reads_raw(node): + found.add((rel, fn.name)) + in_a_function.add(id(node)) + # A read outside every def (module or class level, a lambda there) is named ``. + if any(_reads_raw(n) and id(n) not in in_a_function for n in ast.walk(tree)): + found.add((rel, "")) + return found + + +def _reads_raw(node: ast.AST) -> bool: + return isinstance(node, ast.Attribute) and node.attr == "duration_seconds" + + +def test_only_the_listed_readers_take_the_raw_column(): + found = _raw_reads() + stray = sorted(found - RAW_READERS) + assert not stray, f"read `length_seconds` (a length) instead of the raw column: {stray}" + # A listed reader that stopped reading it leaves the list. + assert not RAW_READERS - found, sorted(RAW_READERS - found) diff --git a/backend/tests/test_ytdlp_logging.py b/backend/tests/test_ytdlp_logging.py index bc98cacc..57ff10f2 100644 --- a/backend/tests/test_ytdlp_logging.py +++ b/backend/tests/test_ytdlp_logging.py @@ -216,7 +216,7 @@ def test_a_line_with_no_url_is_left_alone(sink): def test_the_download_options_carry_the_logger(): - opts = formats.build_ydl_opts({}, "%(id)s.%(ext)s", lambda d: None) + opts = formats.build_ydl_opts({}, "%(id)s.%(ext)s", lambda d: None, egress=False) assert opts["logger"] is formats._YDL_LOG @@ -227,7 +227,7 @@ def test_a_source_adapter_cannot_silence_it(): adapter must not be able to take away the one account of what went wrong that a failed job leaves behind.""" opts = formats.build_ydl_opts( - {}, "%(id)s.%(ext)s", lambda d: None, extra={"logger": None, "quiet": False} + {}, "%(id)s.%(ext)s", lambda d: None, extra={"logger": None, "quiet": False}, egress=False ) assert opts["logger"] is formats._YDL_LOG diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 29015542..27a63d6c 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -568,6 +568,9 @@ of a full-width panel and nothing on the right. the first tap however long the video played). - **The bars stay up while the pointer rests on them** (F22, owner); leaving starts a fresh 3 s. YouTube and Plex Web do the same: before, the row faded out from under a mouse parked on Next. +- **Text over the picture sits on a dark panel** (2026-11-07): loading, stalls, errors and the notices are + readable on any frame. A stall that lasts 5 s on a link measured slower than the stream says "Slow + connection" with both rates. - **Volume is the native `Slider`** (A6); the hand-drawn bar had no hover, focus, keyboard or screen-reader state. A pointer drag gives the focus back, so Space and the arrows keep driving the player. - **The seek bar:** the track stays thin inside a 2.75rem (44px) hit area with `touch-action: none`, so a finger @@ -718,155 +721,160 @@ of a full-width panel and nothing on the right. ## Decision log -| Date | Decision | Chosen | Over | Why | -| ---------- | ---------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 2026-09-24 | Floating button | Back to top: a round 48 glass FAB; "round" = floats over something (media or the page) | A Material 3 rounded square; a labelled pill; the old 64 circle | The common web convention for back to top; round tells it apart from every toolbar button; 64 was oversized | -| 2026-09-24 | Device contract | Tablet portrait + landscape and up; phones out | A 1024px floor with a "too small" message | Portrait tablet use is real, and the app already renders at 820 portrait | -| 2026-09-24 | Card actions on touch | Fewer visible, the rest in a "…" menu | 44px buttons that no longer fit the card | Six 44px targets need more width than a card has | -| 2026-09-24 | Action placement | One menu with several triggers, never gesture-only, a desktop background menu that mirrors the toolbar | Visible buttons for everything | Fewer non-critical buttons on screen, without hiding any function | -| 2026-09-24 | Text selection | Text stays selectable; a click that ends a selection does nothing | Click-to-play on titles | The user must be able to copy any text the app shows | -| 2026-09-24 | A1 Button shape | r8 + a rung per context; glass is a Button variant | r12; a pill | Nested corners (r8 in r12 in r16); the pill already means "a filter or a choice" | -| 2026-09-24 | A2 Icon buttons | Two sizes, r8; a round button only on media | One size; all round | A lone toolbar icon at the row size looks lost; round rows read as tags | -| 2026-09-24 | A3 Touch size | 44 for every control primitive | 36 + gaps; Button only | One rule (HIG 44) instead of two sizes nobody remembers | -| 2026-09-24 | A4 Input | Bordered, a 55% fg-into-bg edge (≥ 3:1) | Filled; today's look as a primitive | The only option that clears WCAG 1.4.11 without a heavy fill on glass | -| 2026-09-24 | A5 Dropdown trigger | Pill in a toolbar, r8 in a form | Pill everywhere; r8 everywhere | Matches the neighbours in each context | -| 2026-09-24 | A6 Checkbox, switch, slider | Native, accent-tinted, the row is the target | Drawn primitives | Free keyboard / screen-reader / forced-colours support; only 3 sites | -| 2026-09-24 | B1 Density | Compact 32 / 28 / 24 by explicit height | Standard 38 / 32 / 26 | The owner's read of the lab; touch is covered separately (A3), and one toolbar row lines up at 28 | -| 2026-09-24 | B2 Scheme set | Midnight, Forest→cyan, Slate→fuchsia, Matrix→lime | Today's six | The accent collided with danger (YouTube), warn (Slate) and success (Forest, Matrix); Midnight and Starship were one scheme twice; 4 of 6 light accents failed 4.5:1 | -| 2026-09-24 | B3 State layers | Text colour at 10% / 16% over anything | Material's 8 / 12%; today's opacity + bg-card | Visible in every scheme and mode (≥ 1.2:1); today's row hover was 1.00:1 on a card | -| 2026-09-24 | B4 Page chrome | One sticky band + fade; filters fold when short | A floating glass bar; everything scrolls | Tabs and filters always in reach, in one place; a keyboard-up tablet keeps its content | -| 2026-09-24 | B5 Surfaces | One class + radius per role | Today's per-site mix | Nested corners and no per-site alpha drift | -| 2026-09-24 | B6 /watch | The app's surfaces, native video controls | Flat and minimal | The page strangers see should look like the product; native controls work on any device | -| 2026-09-24 | B7 Nav rail | Soft tint + edge bar, optional labels (a user setting), slim | Solid accent (recommended); a Material rail | The owner's combination: calmer than a solid fill, labels for who wants them, no width cost | -| 2026-09-24 | B8 Loading | Skeleton by shape, spinner in a control | A spinner everywhere | Nothing jumps when data lands; a long wait doesn't look stuck | -| 2026-09-24 | B9 Fade | Toolbars / headers, panel tabs, dialog edges, chip rows | Thumbnails; buttons | The north-star depth where content scrolls under chrome; not on content or small controls | -| 2026-09-24 | C1 Action classification | The table (lab C1): state + primary at rest, everyday toggles on hover, "…" on touch, destructive into the menu | Visible buttons per site, as today | One action list per object, nothing gesture-only, fewer buttons without losing a function | -| 2026-09-24 | C2 Side panels | A tap opens an overlay drawer; pin is a separate button ≥ 1024 | A tap pins and the content reflows | Below 768 the filters were unreachable; on touch a tap pinned and pushed the grid | -| 2026-09-24 | C3 Players on touch | Tap = show the controls; 44px seek hit area; 44 buttons; the YT description behind ⓘ | Today (tap also pauses; 7.5px seek bar) | Measured live: you couldn't reach the seek bar without pausing, and the hover-only description never opened on a tap | -| 2026-09-24 | C4 Dialogs | Focus by purpose; a pinned footer | Focus on Close; the footer scrolls | A form starts in its first field, and Save stays on screen with the keyboard up | -| 2026-09-24 | C5 Plex player controls | The owner's lean set: no Stop / Restart / ±10 s buttons, Back by position, Next only with a next item, Resume + From the start on the card, touch double-tap ±10 s | Every control in one row (it ran off-screen below 1024); Back by a press counter; ±10 s keyboard-only | The row fits a portrait tablet; the decision to restart is made on the card; a tablet keeps the 10 s skip | -| 2026-09-24 | Glass tuner | Retired; the schemes are the only colour choice | Keep it with ranges clamped to the rules | Free tuning is the drift the constitution exists to stop (owner) | -| 2026-09-24 | Matrix toned down | Moss #86a05a at Midnight's lightness, near-neutral greys; every white-on-accent part → `--accent-fg` | The B2 lime #a3e635 | The owner found the lime neon and tiring; it was the loudest accent (L 0.85, C 0.21), and fixed white parts sat on it at 1.5:1 | -| 2026-09-24 | Field primitives | `Input` / `Textarea` / `InnerInput` + `fieldClass`, `SelectTrigger`, the native controls; the lint rule "no raw ``" | `inputCls` and ~40 hand-styled boxes | Six edges, four radii and two focus looks for one control; the ≥ 3:1 edge held nowhere | -| 2026-09-25 | Surface roles | `SURFACE` roles in `ui/surfaces.ts`; the lint rule "no bare glass class"; menus p-1 with r8 rows | Three tiers paired ad hoc with five radii and three menu paddings | ~140 sites drifted; nested corners only hold if the radius comes with the role | -| 2026-09-25 | Static labels | Flat tonal `Badge`, no edge; a filtering tag is an edged chip | A glass pill with an edge, like the chips | Edged = pressable at a glance; a static tag that looks like a chip gets clicked and does nothing | -| 2026-09-25 | Header search | A 40 glass pill with the field edge (`lg`); what sits inside it is round | 44, r16, a glass box with a faint edge | The one rounded box among pills, and its edge under 3:1; the hero search is the YouTube/Gmail pill | -| 2026-09-25 | Switch row (F12) | The whole settings row toggles (`SwitchRow`; a hand-laid row points a label at the switch) | Only the 36×24 switch toggles | The iOS / Material settings row; the text was a dead click beside the control it names | -| 2026-09-25 | Edges and hovers (F14) | The ≥ 3:1 edge on the field family (fields, the form trigger); labelled pills keep the faint edge and the accent-edge hover | The ≥ 3:1 edge and a state-layer hover on every control | 1.4.11 asks an edge only where nothing else identifies the control; a heavy edge on every chip and tab weighs the whole UI down | -| 2026-09-25 | Rating and live (F16) | Own tokens: `--rating` (theme-scoped gold), `--live`/`--live-fg`, `--rating-media`, `--media-chrome`; palette classes lint-refused | Map them to `--warn` / `--danger` | A gold star and a red LIVE are what the eye expects; as warn/danger they would read as a status. The old amber score was 2:1 on light | -| 2026-09-25 | Fjord trial (B2 exception) | A fifth scheme, temporary: Midnight's neutrals, Playnite's steel accent (#337599 lifted to #5aa1cb) and steel rules; exit = fold into Midnight or remove | Retune Midnight straight away; borrow only the qualities | The owner wants to live with the calmer palette app-wide before it replaces the default; it breaks "own hue family" on purpose, and only until that call | -| 2026-09-25 | Whole-card row (owner) | A card row's B3 layer is the card's `::before` under its content; the open button's hit layer spans the card, the actions sit above it | The name button's own hover, ending at the first badge or button | A row that half-answers the pointer reads as broken; a badge click did nothing | -| 2026-09-25 | Badge tones (F29) | `Badge tone` neutral / accent / warn at 6%; light `--warn` #b45309 → #a14a07; edged label spans lint-refused | Keep edged status pills (a private Badge in Users, Settings "Granted", the suggestions count) | Edged = pressable at a glance; amber-700 was 4.58:1 as text and fell under 4.5 on its own tint | -| 2026-09-26 | Page band built (B4) | Controls only in the band (intros scroll); the fold shows the same row in place; one `h1` per page (`PageHeading` steps the hidden one aside); `SectionLabel` headings; Skip to content | A popover of the folded filters; the page title kept visible | A folded row is still the same controls, nothing duplicated; the rail and the tab title already name the page | -| 2026-09-26 | Fade built (B9) | `fade-below` on sticky toolbars and table headers; the scroller mask where the band sits on the backdrop | A painted band over the backdrop | The backdrop is the colour under the band, so fading the content into it is the same edge | -| 2026-09-26 | Viewport units | `dvh` everywhere (the Plex player stage with S3f) | `vh` / `h-screen` | `100vh` hides the last rows under a tablet browser's toolbar | -| 2026-09-26 | Video actions built (C1) | One list per video (menu, right-click, long-press, "…"); the card keeps watched / save / add to playlist on its picture on hover; download, details, copy, hide, reset in the menu | The six-button row under the card and the hover ⓘ | The menu is the one door every device has; the card loses a line of height and the touch card its row of 28px buttons | -| 2026-09-26 | Plex actions built (C1) | One list per Plex title (play/open, watched, add to playlist, media info, copy title, pin); the poster keeps watched / add / "…" on its picture on hover; seasons get open / watched / add / copy | The hover ⓘ, the pin-only menu, an episode's add button at rest, a collection's "Remove" X on the card | The same door as a video's; the collection grid goes to two columns on a phone so the touch "…" leaves the title its line | -| 2026-09-26 | Playlist actions built (C1) | A playlist row's "…" (play, pin, copy title / link, move, order, remove), Delete + an undo toast; one list per playlist (rename, pin, YouTube open / push / revert, delete) from its rail row and its header's "…" | The row's X column and its confirm; the header's pencil, YouTube link, push, revert and trash buttons | Remove is reversible, so it needs no confirm; the header keeps Play all; a narrow table drops "#" and Channel so the title keeps a width | -| 2026-09-26 | Channel + Plex list actions (C1) | A channel row's "…" (open, pin, YouTube, copy, full history, hide + undo, unsubscribe); a Plex playlist's title / show / season "…", Delete + undo; its header's "…" (rename, delete); an action toast lasts ≥ 8 s | The channel row's full-history, hide and unsubscribe buttons; the Plex list's red X on every row, show and season; the header's pencil and trash | Hiding and removing are reversible, so they need no confirm and the undo needs time; unsubscribe and deleting a playlist keep theirs | -| 2026-09-26 | Download / notification / Scheduler actions (C1) | A download row's "…" (play, share, save, edit, details, re-download, manage sharing, copies, cancel / delete / remove); a library row's Share stays in sight, ahead of save; a notification row's "…" and hover dismiss; the inbox band's and the Scheduler strip's "…" | The download row's five rare buttons and its cancel / delete / remove; the source link's copy button; an activity notice's Find / Undo links; the band's Clear everything; the strip's Purge discovery | The destructive rows keep their confirm, and Clear everything gains one; a row link keeps the browser's own menu; Find in feed drops the tag / search / age filters that could hide its video | -| 2026-09-26 | Channel menu: Tags + Priority (C1) | Tags A–Z as in-place checks; Priority as fixed steps 10 · 5 · 3 · 2 · 1 · 0, the current one ticked; a pick re-sorts at once and the page + focus follow the row; one drawn `MenuCheck` for every checkable row that cannot be an `` | Relative steps (+1 / +5); straight to the top; the stepper only; seven hand-drawn check marks (three box sizes, a tinted box, a × for "on", a trailing tick) | Bubbling a channel up is one pick; fixed levels keep the scale the reader already uses; the list moving where the reader can see it is the feedback | -| 2026-09-26 | Reorder without a drag (WCAG 2.5.7) | Row menus: Top / Up / Down / Bottom (YouTube + Plex playlists); saved views: four buttons with the place beside the selected view (owner: A); side panels in edit mode: an Up / Down pair per group (owner: A), the name wrapping there | A "…" on every saved-view row; a "…" on every panel group; drag only | A visible, one-click path where the list is edited; the edge moves shown off so the buttons never shift | -| 2026-09-26 | Titles in click surfaces | The surface stays a button, flagged `selectable`: its text selects, and the click that ends a selection does nothing; every plain `Pressable` pinned with its reason | Rebuilding each card and row so the title sits outside the button; exceptions for master-detail lists | Chrome selects inside a button once it is `user-select: text` (measured), so one flag on the primitive keeps the keyboard, focus and list walking every surface already has | -| 2026-09-26 | Right-click on a selection | The browser's own menu opens on selected words; beside them the object's menu | A "Copy selection" row in the app menu | The reader selected the words to do something with them; the native menu copies, searches and translates, and changes nothing else | -| 2026-09-26 | Keyboard ring on a flush row | Inside the row (a table row by its tag, a log line, a show header in its clipping card, with the card's corners) | The ring outside, as on a card | Outside it ran under the table head and past the table, and a clipping card cut it away entirely (invisible focus) | -| 2026-09-26 | Fade at rest | The sticky header's fade appears with the first 1rem of scroll (a scroll-driven animation; no fade where unsupported) | A fade that is always drawn | At rest it dimmed the first row and hid its ring's top edge; nothing is under the header yet | -| 2026-09-26 | Narrow playlist table | "#" and Channel hide by the table's own width (under 42rem), not the window's | Hiding below the `sm` window width | Beside the playlist rail a desktop window left the table 432 px and the title 0 px | -| 2026-09-26 | Background menu built | Five list pages (feed, Plex grid, channels, playlist, notifications); the toolbar's sort / view / filters and the band's actions; no destructive row | Every page with a toolbar; the "…" menus copied whole, delete included | The tab-only pages have nothing a menu would add; a blank-area right-click must not be one click from delete | -| 2026-09-26 | Media card anatomy (C) | One frame; the meta line pinned to its bottom; the length a badge on the picture; the Plex card on the same primitive as the video card | A: the year as an overline over the title; B: the year inline after it; a fixed two-line title slot | A one-line title left a hole the eye reads as a missing line; C puts nothing under a variable-height title | -| 2026-09-26 | One Play | `MediaPlayGlyph` (the media button variant) and only where the click plays | The Plex card's hand-painted accent circle; a playable dot on every card | One Play for every card; a dot that only a colour and a tooltip explain says nothing | -| 2026-09-26 | Filtering values on the info page | A value that filters (year, certificate, studio, director) is an edged chip; a fact is flat text | Everything flat, the clickable ones only hover-tinted | The S3d rule: edged = pressable, so the reader sees which values do something | -| 2026-09-26 | Shelf density | On any strip: one title line over one meta line | The grid's two lines and full meta on a strip | The tallest card set the strip's height and left holes in the cards with less to say | -| 2026-09-26 | Compact list row (B) | The title leads; a faint "…" at the end; the toggles float over the date on hover | A leading 128px action column | That column sat empty at rest, a hole at the start of every row | -| 2026-09-26 | Media-info layout (B) | A main column (collections first, crew, cast) beside a facts sidebar; an empty column is not drawn | Every section full width, stacked | The collection is the most used; the keywords' column left a void beside a short details box | -| 2026-09-26 | Mixed Home shelf (A) | Plex titles as wide art beside videos | Posters, with the videos not stretched | Stretching a video card to a poster's height left an ~80px gap | -| 2026-09-27 | Home hero | A card (hover row + Play on the picture, the whole menu, a touch "…"; owner: A); three quarters wide and centred on a wide window; one fact line; the summary at the measure | The hero's pin row alone; full width; an 18rem picture column | The one answer stands in focus, and it offers what the same card offers anywhere | -| 2026-09-27 | Home shelf tiles | A channel / playlist tile: Play, ↗ open, "…" with Open; a show acts on its next episode, ⓘ opens it; the shelf slot 232px | The tile's pin row alone; 200px | A tile without its object's actions read as broken; at 200 the Play touched the hover row | -| 2026-09-27 | Synopsis preview (C) | A ~0.9 s mouse rest opens a passive panel beside the card | The synopsis as the picture's `title` tooltip; an ⓘ to hover; none | One long tooltip line fired anywhere on the art and clashed with the cut title's own tooltip | -| 2026-09-27 | Reading measure | Prose capped at ~65 characters a line (`measure`) | The container's full width | At 110–130 characters a line the eye loses the next line's start | -| 2026-09-24 | Chip family | One `Chip` (filter / token / tag / suggest / ghost) + `DismissibleChip`; nothing under 24; the lint rule "no raw `