diff --git a/backend/alembic/versions/0117_videos_length_sort_indexes.py b/backend/alembic/versions/0117_videos_length_sort_indexes.py new file mode 100644 index 00000000..7d1f1e30 --- /dev/null +++ b/backend/alembic/versions/0117_videos_length_sort_indexes.py @@ -0,0 +1,76 @@ +"""videos: the duration sorts ride an index on the video's length, not on the raw column + +Revision ID: 0117_videos_length_sort_indexes +Revises: 0116_followed_explored_channels +Create Date: 2026-11-07 + +A YouTube length past a day is not a length (`catalog.models.LENGTH_MAX_S`): a 24/7 stream's archive +reports the years it has run. The feed's duration sorts now order by `Video.length_seconds`, which +is NULL for such a video, so "longest first" no longer opens on a stream archive. Migration 0096's +two duration indexes are on the raw column and no longer match that ORDER BY; they are replaced by +the same indexes over the length expression. The expression is spelled out here as SQLAlchemy emits +it, with the limit as a literal (an index matches a query's constant, never a bound parameter); +tests/test_feed_sort_indexes.py fails if the two drift apart. + +CONCURRENTLY (autocommit_block), re-runnable, as in 0096. +""" + +from alembic import op +from sqlalchemy import text + +revision: str = "0117_videos_length_sort_indexes" +down_revision: str | None = "0116_followed_explored_channels" +branch_labels: str | None = None +depends_on: str | None = None + +# Frozen with this migration: `catalog.models.LENGTH_MAX_S` is 24 h. +LENGTH = "(CASE WHEN (duration_seconds > 86400) THEN NULL ELSE duration_seconds END)" + +NEW = { + "ix_videos_length_desc": f"{LENGTH} DESC NULLS LAST, id", + "ix_videos_length_asc": f"{LENGTH} ASC NULLS LAST, id", +} +OLD = { + "ix_videos_duration_desc": "duration_seconds DESC NULLS LAST, id", + "ix_videos_duration_asc": "duration_seconds ASC NULLS LAST, id", +} + + +def _valid(name: str) -> bool | None: + """The index's `indisvalid`, or None when there is no index of that name.""" + row = ( + op.get_bind() + .execute( + text( + "SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid " + "WHERE c.relname = :name" + ), + {"name": name}, + ) + .first() + ) + return None if row is None else bool(row[0]) + + +def _build(name: str, columns: str) -> None: + """CREATE INDEX CONCURRENTLY, re-runnable: an INVALID leftover of a failed build is dropped first + (see 0096 `_build`).""" + if _valid(name) is False: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") + op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} ON videos ({columns})") + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + for name, columns in NEW.items(): + _build(name, columns) + for name in OLD: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + for name, columns in OLD.items(): + _build(name, columns) + for name in NEW: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") diff --git a/backend/app/catalog/models.py b/backend/app/catalog/models.py index 5ea760b9..a55a9e84 100644 --- a/backend/app/catalog/models.py +++ b/backend/app/catalog/models.py @@ -16,10 +16,16 @@ from sqlalchemy import ( String, Text, UniqueConstraint, + and_, + case, func, + literal_column, + null, ) from sqlalchemy.dialects.postgresql import TSVECTOR +from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.sql.elements import ColumnElement from app.db import Base from app.models import TimestampMixin @@ -114,6 +120,20 @@ class Subscription(Base, TimestampMixin): # hidden from feeds/search and revisited by the refresh job until they settle into a VOD. LIVE_OR_UPCOMING = ("live", "upcoming") +# A YouTube length past a day is not a length. A 24/7 stream's archive (`was_live`) reports the years +# the stream has run (measured on prod, 2026-11-06: 122,962,914 s), and one such video made a Watch +# later total read 44,073 hours. YouTube caps an upload at 12 hours, so nothing real is lost. Such a +# length is unknown wherever a length is used: the API sends none (`Video.length_seconds`), sums leave +# it out and count it as `live_without_length`, and the duration filter, sort and Home budget never +# match it. The raw `duration_seconds` stays for what it does prove — that the video is not a Short — +# and for the jobs that ask whether YouTube has answered yet (test_video_length holds who reads it). +LENGTH_MAX_S = 24 * 3600 + + +def known_length(seconds: int | None) -> int | None: + """`seconds` as a video's length, or None when it is not one (see LENGTH_MAX_S).""" + return seconds if seconds is not None and seconds <= LENGTH_MAX_S else None + class Video(Base, TimestampMixin): """A YouTube video. Shared across users; per-user state lives elsewhere.""" @@ -141,8 +161,8 @@ class Video(Base, TimestampMixin): ) thumbnail_url: Mapped[str | None] = mapped_column(String(1024)) # The feed's views / duration / title sorts each ride a migration-defined index in their exact - # ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to - # `feedquery.order_by` by test_feed_sort_indexes. + # ORDER BY, one per direction (and per ICU collation for title): migration 0096 (the duration + # pair since 0117, on `length_seconds`), held to `feedquery.order_by` by test_feed_sort_indexes. duration_seconds: Mapped[int | None] = mapped_column(Integer) view_count: Mapped[int | None] = mapped_column(BigInteger) like_count: Mapped[int | None] = mapped_column(BigInteger) @@ -198,6 +218,33 @@ class Video(Base, TimestampMixin): unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) unavailable_reason: Mapped[str | None] = mapped_column(String(24)) + @hybrid_property + def length_seconds(self) -> int | None: + """The video's length, or None when it is unknown — not fetched yet, or not a length at all + (`LENGTH_MAX_S`). What every reader of a length reads.""" + return known_length(self.duration_seconds) + + @length_seconds.inplace.expression + @classmethod + def _length_seconds_expression(cls) -> ColumnElement[int | None]: + # The limit is a literal, not a bound parameter: the duration sort indexes (migration 0117) + # are built on this very expression, and Postgres matches an index to a query's constant only. + return case( + (cls.duration_seconds > literal_column(str(LENGTH_MAX_S)), null()), + else_=cls.duration_seconds, + ) + + @hybrid_property + def live_without_length(self) -> bool: + """A live video (on air, upcoming, a premiere or an archive) whose length is unknown: what a + total leaves out and names as "+N live".""" + return self.live_status != "none" and self.length_seconds is None + + @live_without_length.inplace.expression + @classmethod + def _live_without_length_expression(cls) -> ColumnElement[bool]: + return and_(cls.live_status != "none", cls.length_seconds.is_(None)) + # Weighted full-text search document (DB-generated, never written by the app): title (A) > # creator keywords + search queries (B) > truncated description (C). ts_rank honours the # weights so title matches outrank description matches. Backed by a GIN index (migration diff --git a/backend/app/catalog/queries.py b/backend/app/catalog/queries.py index 63b2caee..1955f458 100644 --- a/backend/app/catalog/queries.py +++ b/backend/app/catalog/queries.py @@ -44,7 +44,8 @@ def channel_titles(db: Session, ids: set[str]) -> dict[str, str]: def channel_aggregates(channel_ids: list[str]) -> Select: - """Per-channel stored count, last upload and total duration, in one grouped pass. Every column it + """Per-channel stored count, last upload, total duration and the live videos the total cannot + hold (`Video.live_without_length`), in one grouped pass. Every column it reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index alone; a test holds that (test_channel_list_index_only).""" return ( @@ -52,15 +53,19 @@ def channel_aggregates(channel_ids: list[str]) -> Select: Video.channel_id, func.count(Video.id), func.max(Video.published_at), - func.coalesce(func.sum(Video.duration_seconds), 0), + func.coalesce(func.sum(Video.length_seconds), 0), + func.count().filter(Video.live_without_length), ) .where(Video.channel_id.in_(channel_ids)) .group_by(Video.channel_id) ) -def aggregates(db: Session, channel_ids: list[str]) -> list[tuple[str, int, datetime | None, int]]: - """`channel_aggregates`, run: (channel id, stored videos, last upload, total seconds).""" +def aggregates( + db: Session, channel_ids: list[str] +) -> list[tuple[str, int, datetime | None, int, int]]: + """`channel_aggregates`, run: (channel id, stored videos, last upload, total seconds, live + videos without a length).""" if not channel_ids: return [] return list(db.execute(channel_aggregates(channel_ids)).tuples()) diff --git a/backend/app/downloads/service.py b/backend/app/downloads/service.py index e2e80910..97073d99 100644 --- a/backend/app/downloads/service.py +++ b/backend/app/downloads/service.py @@ -153,7 +153,7 @@ def populate_from_catalog(db: Session, asset: MediaAsset) -> bool: return False asset.title = video.title asset.thumbnail_url = video.thumbnail_url - asset.duration_s = video.duration_seconds + asset.duration_s = video.length_seconds if video.published_at: asset.upload_date = video.published_at.date() channel = db.get(Channel, video.channel_id) @@ -337,7 +337,7 @@ def _clip_source_facts(db: Session, spec: dict) -> dict: "title": video.title, "uploader": channel.title if channel else None, "thumbnail_url": video.thumbnail_url, - "length_s": video.duration_seconds, + "length_s": video.length_seconds, } item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"]) if item is None: diff --git a/backend/app/feedquery.py b/backend/app/feedquery.py index c703846c..173c806e 100644 --- a/backend/app/feedquery.py +++ b/backend/app/feedquery.py @@ -68,7 +68,7 @@ def feed_columns(user: User, state) -> list: Channel.handle.label("channel_handle"), Video.published_at, Video.thumbnail_url, - Video.duration_seconds, + Video.length_seconds.label("length_seconds"), Video.view_count, Video.is_short, Video.live_status, @@ -88,7 +88,7 @@ def serialize(row) -> dict: "channel_url": _channel_url(row.channel_id, row.channel_handle), "published_at": iso(row.published_at), "thumbnail_url": row.thumbnail_url, - "duration_seconds": row.duration_seconds, + "duration_seconds": row.length_seconds, "view_count": row.view_count, "is_short": row.is_short, "live_status": row.live_status, @@ -260,9 +260,9 @@ def filtered_query( # OR across the picked channels; every other filter still applies on top. query = query.where(Video.channel_id.in_(channel_ids)) if min_duration is not None: - query = query.where(Video.duration_seconds >= min_duration) + query = query.where(Video.length_seconds >= min_duration) if max_duration is not None: - query = query.where(Video.duration_seconds <= max_duration) + query = query.where(Video.length_seconds <= max_duration) if max_age_days is not None: cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400 query = query.where(Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc)) @@ -503,10 +503,10 @@ def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) -> "views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}], "views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}], "duration_desc": [ - {"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"} + {"expr": Video.length_seconds, "asc": False, "nullable": True, "kind": "num"} ], "duration_asc": [ - {"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"} + {"expr": Video.length_seconds, "asc": True, "nullable": True, "kind": "num"} ], "title": [{"expr": title, "asc": True, "nullable": True, "kind": "str"}], "title_desc": [{"expr": title, "asc": False, "nullable": True, "kind": "str"}], diff --git a/backend/app/reader/queries.py b/backend/app/reader/queries.py index b5e9c026..2bf2f4d7 100644 --- a/backend/app/reader/queries.py +++ b/backend/app/reader/queries.py @@ -146,12 +146,17 @@ def item_counts(db: Session, ids: list[int]) -> dict[int, int]: ) -def durations(db: Session, ids: list[int]) -> dict[int, int]: - """Each playlist's running time: the sum of its videos' lengths.""" +def durations(db: Session, ids: list[int]) -> dict[int, tuple[int, int]]: + """Each playlist's running time — the sum of its videos' known lengths — and how many live videos + it cannot hold (`Video.live_without_length`).""" return { - pid: int(total or 0) - for pid, total in db.execute( - select(PlaylistItem.playlist_id, func.coalesce(func.sum(Video.duration_seconds), 0)) + pid: (int(total or 0), int(live or 0)) + for pid, total, live in db.execute( + select( + PlaylistItem.playlist_id, + func.coalesce(func.sum(Video.length_seconds), 0), + func.count().filter(Video.live_without_length), + ) .join(Video, Video.id == PlaylistItem.video_id) .where(PlaylistItem.playlist_id.in_(ids)) .group_by(PlaylistItem.playlist_id) @@ -191,11 +196,16 @@ def covers(db: Session, ids: list[int]) -> dict[int, str | None]: ) -def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None]: - """One playlist's item count, running time and cover. An outer join, so an empty list still - answers (0, 0, None).""" - count, total = db.execute( - select(func.count(PlaylistItem.id), func.coalesce(func.sum(Video.duration_seconds), 0)) +def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, int, str | None]: + """One playlist's item count, running time, live videos the running time cannot hold + (`Video.live_without_length`) and cover. An outer join, so an empty list still answers + (0, 0, 0, None).""" + count, total, live = db.execute( + select( + func.count(PlaylistItem.id), + func.coalesce(func.sum(Video.length_seconds), 0), + func.count().filter(Video.live_without_length), + ) .select_from(PlaylistItem) .outerjoin(Video, Video.id == PlaylistItem.video_id) .where(PlaylistItem.playlist_id == playlist_id) @@ -207,7 +217,7 @@ def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None .order_by(PlaylistItem.position) .limit(1) ) - return count or 0, total or 0, cover + return count or 0, total or 0, live or 0, cover def playlist_rows(db: Session, user: User, playlist_id: int) -> list: diff --git a/backend/app/routes/channels.py b/backend/app/routes/channels.py index beedfa9e..33e80a47 100644 --- a/backend/app/routes/channels.py +++ b/backend/app/routes/channels.py @@ -46,11 +46,12 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_ # computed here — the manager table dropped its N/S/L column.) channel_ids = [c.id for _, c in rows] agg: dict[str, dict] = {} - for cid, n, last_up, total_dur in queries.aggregates(db, channel_ids): + for cid, n, last_up, total_dur, live in queries.aggregates(db, channel_ids): agg[cid] = { "stored": n, "last_video_at": iso(last_up), "total_duration_seconds": int(total_dur or 0), + "live_without_length": live, } # Channels already in the *global* deep-backfill queue — i.e. at least one user (any @@ -68,6 +69,7 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_ "stored_videos": (agg.get(ch.id) or {}).get("stored", 0), "last_video_at": (agg.get(ch.id) or {}).get("last_video_at"), "total_duration_seconds": (agg.get(ch.id) or {}).get("total_duration_seconds", 0), + "live_without_length": (agg.get(ch.id) or {}).get("live_without_length", 0), "priority": sub.priority, "hidden": sub.hidden, "deep_requested": sub.deep_requested, diff --git a/backend/app/routes/feed.py b/backend/app/routes/feed.py index 43581896..237cd8f7 100644 --- a/backend/app/routes/feed.py +++ b/backend/app/routes/feed.py @@ -10,6 +10,7 @@ from sqlalchemy.orm.exc import StaleDataError from app import jobruns, quota, scheduler from app.catalog import queries as catalog_queries, service as catalog_service +from app.catalog.models import known_length from app.auth import current_user from app.db import get_db, release_read_connection from app.mailtext import lang_for_user @@ -287,7 +288,7 @@ def get_video_detail( "channel_title": v.channel.title if v.channel else None, "published_at": iso(v.published_at), "view_count": v.view_count, - "duration_seconds": v.duration_seconds, + "duration_seconds": v.length_seconds, } # The DB path above is free; the off-catalog YouTube lookup below spends a quota unit. @@ -315,8 +316,8 @@ def get_video_detail( "channel_title": snippet.get("channelTitle"), "published_at": snippet.get("publishedAt"), "view_count": int(views) if views is not None else None, - "duration_seconds": parse_iso8601_duration( - items[0].get("contentDetails", {}).get("duration") + "duration_seconds": known_length( + parse_iso8601_duration(items[0].get("contentDetails", {}).get("duration")) ), } diff --git a/backend/app/routes/playlists.py b/backend/app/routes/playlists.py index 79ac345a..c2a1fe8b 100644 --- a/backend/app/routes/playlists.py +++ b/backend/app/routes/playlists.py @@ -95,10 +95,13 @@ def _video_or_404(db: Session, video_id: str) -> None: def _summary( pl: "Playlist", count: int, - total_duration: int = 0, + duration: tuple[int, int] = (0, 0), has_video: bool | None = None, cover: str | None = None, ) -> dict: + """`duration` is `queries.durations`'s pair: the known running time, and the live videos it + cannot hold — the "+N live" beside it.""" + total_duration, live = duration out = { "id": pl.id, "name": pl.name, @@ -108,6 +111,7 @@ def _summary( "dirty": pl.dirty, "item_count": count, "total_duration_seconds": total_duration, + "live_without_length": live, "cover_thumbnail": cover, # When this list last CHANGED (rename, add/remove/move/reorder) — the rail's # "recently changed first" sort reads it. See `service.touch` for why it isn't automatic. @@ -138,7 +142,7 @@ def list_playlists( _summary( p, counts.get(p.id, 0), - durations.get(p.id, 0), + durations.get(p.id, (0, 0)), (p.id in member) if contains else None, covers.get(p.id), ) @@ -363,8 +367,8 @@ def rename_playlist( raise _refused(400, "name_required") service.rename_playlist(db, pl, name) db.commit() - count, total, cover = queries.playlist_totals(db, pl.id) - return _summary(pl, count, total, cover=cover) + count, total, live, cover = queries.playlist_totals(db, pl.id) + return _summary(pl, count, (total, live), cover=cover) @router.delete("/{playlist_id}") diff --git a/backend/tests/test_migration_0110.py b/backend/tests/test_migration_0110.py index ba3677b8..82fe996a 100644 --- a/backend/tests/test_migration_0110.py +++ b/backend/tests/test_migration_0110.py @@ -45,6 +45,9 @@ def test_an_interval_override_moves_into_sysconfig_and_back(db): } assert conn.execute(text("SELECT to_regclass('scheduler_settings')")).scalar() is None assert scheduler.load_intervals(db)["rss_poll"] == 25 + # The way down rebuilds an index CONCURRENTLY (0117), which waits for every open transaction — + # the read above left this session's open. + db.rollback() command.downgrade(cfg, "0109_job_runs") try: diff --git a/backend/tests/test_reader_service.py b/backend/tests/test_reader_service.py index 647c6825..7d54e648 100644 --- a/backend/tests/test_reader_service.py +++ b/backend/tests/test_reader_service.py @@ -252,7 +252,7 @@ def test_an_add_is_idempotent_and_lands_where_asked(db, make_user): db.commit() assert _ids(db, pl) == ["c", "b"] assert queries.item_counts(db, [pl.id]) == {pl.id: 2} - assert queries.durations(db, [pl.id]) == {pl.id: 120} + assert queries.durations(db, [pl.id]) == {pl.id: (120, 0)} assert queries.containing(db, [pl.id], "b") == {pl.id} @@ -294,7 +294,7 @@ def test_a_rename_and_a_delete(db, make_user): service.add_item(db, pl, "a", mark_dirty=False) service.rename_playlist(db, pl, "New") db.commit() - assert queries.playlist_totals(db, pl.id) == (1, 60, None) + assert queries.playlist_totals(db, pl.id) == (1, 60, 0, None) assert pl.name == "New" service.delete_playlist(db, pl) db.commit() diff --git a/backend/tests/test_video_length.py b/backend/tests/test_video_length.py new file mode 100644 index 00000000..85ecebc5 --- /dev/null +++ b/backend/tests/test_video_length.py @@ -0,0 +1,182 @@ +"""A YouTube length past a day is unknown, everywhere a length is used (`catalog.models.LENGTH_MAX_S`). + +Measured on prod (2026-11-06): two 24/7 streams' archives report 122,962,914 s and 34,762,459 s, and the +owner's Watch later total read "44,073 hr 55 min". Such a video now has no length on any API answer, the +totals leave it out and count it as `live_without_length`, and the duration filter and sort pass it over +like a video whose length is not fetched yet. An AST check holds who may still read the raw column. +DB lane (the AST check needs none). +""" + +import ast +from datetime import datetime, timezone +from pathlib import Path + +from sqlalchemy.dialects import postgresql + +from app.catalog.models import LENGTH_MAX_S, Channel, Subscription, Video, known_length +from app.reader.models import Playlist, PlaylistItem + +YEARS = 122_962_914 # the measured archive +HOUR = 3600 + + +def _seed(db, user): + db.add(Channel(id="UClen", title="Len")) + db.flush() + db.add(Subscription(user_id=user.id, channel_id="UClen", priority=0, hidden=False)) + rows = [ + ("short", 600, "none"), + ("long", 2 * HOUR, "none"), + ("day", LENGTH_MAX_S, "was_live"), # exactly a day still is a length + ("archive", YEARS, "was_live"), + ("onair", None, "live"), + ("unfetched", None, "none"), # not live: not counted as "+N live" + ] + for i, (vid, seconds, live) in enumerate(rows): + db.add( + Video( + id=vid, + channel_id="UClen", + published_at=datetime(2024, 1, i + 1, tzinfo=timezone.utc), + title=vid, + duration_seconds=seconds, + live_status=live, + is_short=False, + via_search=False, + via_explore=False, + ) + ) + pl = Playlist(user_id=user.id, name="Later") + db.add(pl) + db.flush() + for i, (vid, _s, _l) in enumerate(rows): + db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=i)) + db.commit() + return pl + + +KNOWN_SUM = 600 + 2 * HOUR + LENGTH_MAX_S + + +def test_the_rule(): + assert known_length(None) is None + assert known_length(LENGTH_MAX_S) == LENGTH_MAX_S + assert known_length(LENGTH_MAX_S + 1) is None + assert known_length(YEARS) is None + + +def test_the_sql_length_carries_its_limit_as_a_literal(): + # The duration sort indexes (migration 0117) are built on this expression with 86400 in it. A + # bound parameter matches no index once the driver reuses a prepared, generic plan, and the + # sort-index test cannot see that: it compiles with literal binds. + compiled = Video.length_seconds.expression.compile(dialect=postgresql.dialect()) + assert compiled.params == {} + assert f"> {LENGTH_MAX_S}" in str(compiled) + + +def test_a_years_long_archive_has_no_length_in_the_feed_or_a_playlist(db, make_user, as_user): + user = make_user() + pl = _seed(db, user) + client = as_user(user) + + feed = client.get("/api/feed?show=all&include_live=true&limit=50") + assert feed.status_code == 200 + by_id = {v["id"]: v["duration_seconds"] for v in feed.json()["items"]} + assert by_id["archive"] is None + assert by_id["day"] == LENGTH_MAX_S + + detail = client.get(f"/api/playlists/{pl.id}") + assert detail.status_code == 200 + items = {v["id"]: v["duration_seconds"] for v in detail.json()["items"]} + assert items["archive"] is None + + one = client.get("/api/videos/archive") + assert one.status_code == 200 + assert one.json()["duration_seconds"] is None + + +def test_the_totals_leave_it_out_and_count_the_live_ones(db, make_user, as_user): + user = make_user() + pl = _seed(db, user) + client = as_user(user) + + listed = client.get("/api/playlists") + assert listed.status_code == 200 + (summary,) = [p for p in listed.json() if p["id"] == pl.id] + assert summary["total_duration_seconds"] == KNOWN_SUM + assert summary["live_without_length"] == 2 # the archive and the one on air + + renamed = client.patch(f"/api/playlists/{pl.id}", json={"name": "Later 2"}) + assert renamed.status_code == 200 + assert renamed.json()["total_duration_seconds"] == KNOWN_SUM + assert renamed.json()["live_without_length"] == 2 + + channels = client.get("/api/channels") + assert channels.status_code == 200 + (ch,) = channels.json() + assert ch["total_duration_seconds"] == KNOWN_SUM + assert ch["live_without_length"] == 2 + + +def test_the_duration_filter_and_sort_pass_it_over(db, make_user, as_user): + user = make_user() + _seed(db, user) + client = as_user(user) + base = "/api/feed?show=all&include_live=true&limit=50" + + longer = client.get(f"{base}&min_duration={HOUR}") + assert longer.status_code == 200 + assert {v["id"] for v in longer.json()["items"]} == {"long", "day"} + + longest = client.get(f"{base}&sort=duration_desc") + assert longest.status_code == 200 + order = [v["id"] for v in longest.json()["items"]] + assert order[:3] == ["day", "long", "short"] + assert set(order[3:]) == {"archive", "onair", "unfetched"} + + +# Who may read `duration_seconds` itself, by (file, function): what each one asks is not "how long". +RAW_READERS = { + # The column is defined and turned into a length here. + ("catalog/models.py", "length_seconds"), + ("catalog/models.py", "_length_seconds_expression"), + # Ingest writes it; the live refresh asks whether YouTube has answered for an archive yet; the + # Shorts classification needs only "longer than a Short", which a years-long value proves. + ("sync/videos.py", "apply_video_details"), + ("sync/videos.py", "refresh_live"), + ("sync/videos.py", "run_shorts_classification"), + ("catalog/search.py", "classify_shorts"), + ("catalog/queries.py", "work_queue"), + # Not a video's column: the player's own report, and the watch history's looked-up lengths, which + # count a watched video as more than a glance (a watched archive is that too). + ("routes/_common.py", "kept_position"), + ("routes/plex/playback.py", "item_progress"), + ("reader/queries.py", "_counted_channels"), + ("reader/queries.py", "watched_channels"), + ("reader/queries.py", "watched_channel_videos"), +} + + +def _raw_reads() -> set[tuple[str, str]]: + root = Path(__file__).resolve().parents[1] / "app" + found: set[tuple[str, str]] = set() + for path in root.rglob("*.py"): + if "alembic" in path.parts: + continue + rel = path.relative_to(root).as_posix() + tree = ast.parse(path.read_text(encoding="utf-8")) + for fn in ast.walk(tree): + if not isinstance(fn, ast.FunctionDef): + continue + for node in ast.walk(fn): + if isinstance(node, ast.Attribute) and node.attr == "duration_seconds": + found.add((rel, fn.name)) + return found + + +def test_only_the_listed_readers_take_the_raw_column(): + found = _raw_reads() + stray = sorted(found - RAW_READERS) + assert not stray, f"read `length_seconds` (a length) instead of the raw column: {stray}" + # A listed reader that stopped reading it leaves the list. + assert not RAW_READERS - found, sorted(RAW_READERS - found) diff --git a/frontend/src/features/video/liveWithoutLength.test.ts b/frontend/src/features/video/liveWithoutLength.test.ts new file mode 100644 index 00000000..44819ba4 --- /dev/null +++ b/frontend/src/features/video/liveWithoutLength.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import type { TFunction } from "i18next"; +import { isLiveWithoutLength, withLiveCount } from "./liveWithoutLength"; + +const t = ((key: string, o: { count: number }) => `${key}:${o.count}`) as unknown as TFunction; + +describe("a total and the live videos it cannot hold", () => { + it("counts a live video only when the server sent no length", () => { + expect(isLiveWithoutLength({ duration_seconds: null, live_status: "was_live" })).toBe(true); + expect(isLiveWithoutLength({ duration_seconds: null, live_status: "live" })).toBe(true); + expect(isLiveWithoutLength({ duration_seconds: 5400, live_status: "was_live" })).toBe(false); + // Not fetched yet, and not live: left out of the sum like before, not named as live. + expect(isLiveWithoutLength({ duration_seconds: null, live_status: "none" })).toBe(false); + }); + + it("names them beside the sum, or alone when nothing summed", () => { + expect(withLiveCount("12 hr 5 min", 2, t)).toBe("12 hr 5 min card.liveWithoutLength:2"); + expect(withLiveCount("", 1, t)).toBe("card.liveWithoutLength:1"); + expect(withLiveCount("12 hr 5 min", 0, t)).toBe("12 hr 5 min"); + }); +}); diff --git a/frontend/src/features/video/liveWithoutLength.ts b/frontend/src/features/video/liveWithoutLength.ts new file mode 100644 index 00000000..b086b616 --- /dev/null +++ b/frontend/src/features/video/liveWithoutLength.ts @@ -0,0 +1,18 @@ +// A total of video lengths and the live videos it cannot hold. The server decides what a length is +// (a YouTube "length" past a day is a 24/7 stream's archive reporting the years it has run, and the +// API sends none for it: `catalog.models.LENGTH_MAX_S`), so this side keeps no copy of that limit. It +// only names what a sum left out: a live video — on air, upcoming, a premiere or an archive — with no +// length, as the server's `Video.live_without_length` counts it for the totals it sums itself. +import type { TFunction } from "i18next"; +import type { Video } from "../../lib/api/feed"; + +export function isLiveWithoutLength(v: Pick): boolean { + return v.duration_seconds == null && v.live_status !== "none"; +} + +/** "12 hr 5 min +2 live": `total` is the formatted sum ("" when nothing summed), `live` the count + * it left out. Empty when both are. */ +export function withLiveCount(total: string, live: number, t: TFunction): string { + const suffix = live > 0 ? t("card.liveWithoutLength", { count: live }) : ""; + return [total, suffix].filter(Boolean).join(" "); +} diff --git a/frontend/src/i18n/locales/en/card.json b/frontend/src/i18n/locales/en/card.json index e566dbed..168a97db 100644 --- a/frontend/src/i18n/locales/en/card.json +++ b/frontend/src/i18n/locales/en/card.json @@ -16,6 +16,8 @@ "restart": "Restart", "play": "Play", "stream": "stream", + "liveWithoutLength_one": "+{{count}} live", + "liveWithoutLength_other": "+{{count}} live", "views": "views", "info": "Details", "noDescription": "No description", diff --git a/frontend/src/i18n/locales/hu/card.json b/frontend/src/i18n/locales/hu/card.json index 80d8a9b8..ceb2e26b 100644 --- a/frontend/src/i18n/locales/hu/card.json +++ b/frontend/src/i18n/locales/hu/card.json @@ -16,6 +16,8 @@ "restart": "Elölről", "play": "Lejátszás", "stream": "közvetítés", + "liveWithoutLength_one": "+{{count}} élő", + "liveWithoutLength_other": "+{{count}} élő", "views": "megtekintés", "info": "Részletek", "noDescription": "Nincs leírás", diff --git a/frontend/src/lib/api/channels.ts b/frontend/src/lib/api/channels.ts index 7ca3c755..be9626c1 100644 --- a/frontend/src/lib/api/channels.ts +++ b/frontend/src/lib/api/channels.ts @@ -77,6 +77,8 @@ export interface ManagedChannel { * subscribe made here. */ added_at: string; total_duration_seconds: number; + /** Live videos the total cannot hold (no length): the "+N live" beside it. */ + live_without_length: number; priority: number; hidden: boolean; deep_requested: boolean; diff --git a/frontend/src/lib/api/playlists.ts b/frontend/src/lib/api/playlists.ts index 4150d508..2dfb2cea 100644 --- a/frontend/src/lib/api/playlists.ts +++ b/frontend/src/lib/api/playlists.ts @@ -17,6 +17,8 @@ export interface Playlist { dirty: boolean; // linked local playlist has edits not yet pushed to YouTube item_count: number; total_duration_seconds: number; + /** Live videos the total cannot hold (no length): the "+N live" beside it. */ + live_without_length: number; cover_thumbnail: string | null; updated_at: string; // ISO; when the list last CHANGED (rename, add/remove/move/reorder) has_video?: boolean; // only set when listed with ?contains= diff --git a/frontend/src/modules/channels/channelColumns.dom.test.tsx b/frontend/src/modules/channels/channelColumns.dom.test.tsx index 80104292..eb2b6b05 100644 --- a/frontend/src/modules/channels/channelColumns.dom.test.tsx +++ b/frontend/src/modules/channels/channelColumns.dom.test.tsx @@ -28,6 +28,7 @@ const ch = (over: Partial): ManagedChannel => ({ last_video_at: null, added_at: "2026-06-10T08:00:00+00:00", total_duration_seconds: 0, + live_without_length: 0, priority: 0, hidden: false, deep_requested: false, diff --git a/frontend/src/modules/channels/channelColumns.tsx b/frontend/src/modules/channels/channelColumns.tsx index ab2b9abc..ee7f6939 100644 --- a/frontend/src/modules/channels/channelColumns.tsx +++ b/frontend/src/modules/channels/channelColumns.tsx @@ -11,6 +11,7 @@ import { ChannelTagsControl, type SuggestedTag } from "../../features/channel/Ch import { StoredCell } from "./channelCells"; import { isFullySynced } from "./channelSync"; import { RelativeTime } from "../../components/ui/RelativeTime"; +import { withLiveCount } from "../../features/video/liveWithoutLength"; import type { FilterChip } from "./TagFilterChips"; /** The shared "subscribers" DataTable column (right-aligned, sortable, em-dash for null) — identical @@ -236,6 +237,10 @@ export function buildChannelColumns(deps: ChannelColumnDeps): Column ( {formatTotalHours(c.total_duration_seconds, lang)} + {/* Its own line: the column is sized for the sum alone. */} + {c.live_without_length > 0 && ( + {withLiveCount("", c.live_without_length, t)} + )} ), }, diff --git a/frontend/src/modules/channels/useChannelActions.dom.test.tsx b/frontend/src/modules/channels/useChannelActions.dom.test.tsx index 66d0bd32..1525590d 100644 --- a/frontend/src/modules/channels/useChannelActions.dom.test.tsx +++ b/frontend/src/modules/channels/useChannelActions.dom.test.tsx @@ -61,6 +61,7 @@ const ch = (over: Partial = {}): ManagedChannel => ({ last_video_at: null, added_at: "2026-06-10T08:00:00+00:00", total_duration_seconds: 0, + live_without_length: 0, priority: 0, hidden: false, deep_requested: false, diff --git a/frontend/src/modules/playlists/Playlists.dom.test.tsx b/frontend/src/modules/playlists/Playlists.dom.test.tsx index 3e3b2dcf..67834d23 100644 --- a/frontend/src/modules/playlists/Playlists.dom.test.tsx +++ b/frontend/src/modules/playlists/Playlists.dom.test.tsx @@ -134,6 +134,7 @@ const vid = (id: string, position_seconds: number): Video => status: "new", position_seconds, duration_seconds: 3000, + live_status: "none", published_at: "2026-01-01T00:00:00Z", watch_url: `https://www.youtube.com/watch?v=${id}`, }) as unknown as Video; @@ -998,6 +999,17 @@ describe("Playlists · the header", () => { expect(await meta()).toBe("playlists.itemCount"); }); + it("names the live videos the sum cannot hold beside it", async () => { + // A 24/7 stream's archive comes with no length (the server's rule): it is named, not summed. + const archive = (id: string) => ({ ...untimed(id), live_status: "was_live" }) as Video; + mount({ 1: [vid("A", 0), archive("B")] }); + expect(await meta()).toBe("playlists.itemCount · 50 min card.liveWithoutLength"); + cleanup(); + + mount({ 1: [archive("C")] }); + expect(await meta()).toBe("playlists.itemCount · card.liveWithoutLength"); + }); + it("offers the view controls only once there is more than one row to arrange", async () => { mount({ 1: [vid("A", 0)] }); await screen.findByText("Video A"); diff --git a/frontend/src/modules/playlists/Playlists.tsx b/frontend/src/modules/playlists/Playlists.tsx index 9a35ec45..4603d1a3 100644 --- a/frontend/src/modules/playlists/Playlists.tsx +++ b/frontend/src/modules/playlists/Playlists.tsx @@ -68,6 +68,7 @@ import { playlistViewParamUpdates, } from "../../lib/playlistsUrlState"; import { formatRuntime } from "../../lib/format"; +import { isLiveWithoutLength, withLiveCount } from "../../features/video/liveWithoutLength"; import { applyView, sameOrder } from "../../lib/playlistView"; import { useOpenLinkedPlayer } from "../../features/player/useOpenLinkedPlayer"; const PlayerModal = lazy(() => import("../../features/player/PlayerModal")); @@ -565,11 +566,13 @@ function PlaylistDetail() { }, [items]); // Total playtime of the whole playlist (sums every item, matching the itemCount beside it — not - // the filtered `shown` set). Items with an unknown duration count as 0; hidden when nothing sums. + // the filtered `shown` set). Items with an unknown duration count as 0, and a live one is named + // beside the sum instead ("+2 live"); hidden when there is neither. const totalSeconds = useMemo( () => items.reduce((acc, v) => acc + (v.duration_seconds ?? 0), 0), [items], ); + const liveCount = useMemo(() => items.filter(isLiveWithoutLength).length, [items]); // The row's "…" opens the row menu under itself. Which row's menu is open is part of the // cell (aria-expanded), so the columns rebuild when it changes — a cheap array of closures. @@ -1619,7 +1622,8 @@ function PlaylistDetail() { > {t("playlists.itemCount", { count: items.length })} - {totalSeconds > 0 && ` · ${formatRuntime(totalSeconds, lang)}`} + {(totalSeconds > 0 || liveCount > 0) && + ` · ${withLiveCount(formatRuntime(totalSeconds, lang), liveCount, t)}`} {filterActive && ` · ${t("playlists.filteredCount", { count: shown.length })}`} diff --git a/frontend/src/modules/playlists/PlaylistsRail.drop.dom.test.tsx b/frontend/src/modules/playlists/PlaylistsRail.drop.dom.test.tsx index f308cf54..587396a1 100644 --- a/frontend/src/modules/playlists/PlaylistsRail.drop.dom.test.tsx +++ b/frontend/src/modules/playlists/PlaylistsRail.drop.dom.test.tsx @@ -75,6 +75,7 @@ const pl = (id: number, name: string) => yt_playlist_id: null, item_count: 1, total_duration_seconds: 0, + live_without_length: 0, cover_thumbnail: null, dirty: false, }) as unknown as Playlist; diff --git a/frontend/src/modules/playlists/PlaylistsRail.rename.dom.test.tsx b/frontend/src/modules/playlists/PlaylistsRail.rename.dom.test.tsx index ff2e5f38..ab67642f 100644 --- a/frontend/src/modules/playlists/PlaylistsRail.rename.dom.test.tsx +++ b/frontend/src/modules/playlists/PlaylistsRail.rename.dom.test.tsx @@ -80,6 +80,7 @@ const alpha = { source: "local", item_count: 3, total_duration_seconds: 0, + live_without_length: 0, cover_thumbnail: null, dirty: false, } as unknown as Playlist; diff --git a/frontend/src/modules/playlists/PlaylistsRail.tsx b/frontend/src/modules/playlists/PlaylistsRail.tsx index 5be62d58..4edac479 100644 --- a/frontend/src/modules/playlists/PlaylistsRail.tsx +++ b/frontend/src/modules/playlists/PlaylistsRail.tsx @@ -8,6 +8,7 @@ import { YouTubeIcon } from "../../lib/brandIcons"; import { api, type Playlist } from "../../lib/api"; import { compareText } from "../../lib/collate"; import { formatRuntime } from "../../lib/format"; +import { withLiveCount } from "../../features/video/liveWithoutLength"; import { msg, notify } from "../../lib/notifications"; import { notifyYouTubeActionError } from "../../lib/youtubeErrors"; import { LS } from "../../lib/storage"; @@ -378,8 +379,12 @@ export default function PlaylistsRail() {
{t("playlists.itemCount", { count: pl.item_count })} - {pl.total_duration_seconds > 0 && - ` · ${formatRuntime(pl.total_duration_seconds, lang)}`} + {(pl.total_duration_seconds > 0 || pl.live_without_length > 0) && + ` · ${withLiveCount( + formatRuntime(pl.total_duration_seconds, lang), + pl.live_without_length, + t, + )}`}