fix(catalog): a 24/7 stream archive's years-long length is unknown, not summed or sorted by
YouTube reports a 24/7 stream's archive with the years it has run (prod: 122,962,914 s), and one
such video made the Watch later total read 44,073 hours. A length past a day is now no length
(`Video.length_seconds`, LENGTH_MAX_S): the API sends none, the playlist and channel totals
leave it out and count it as `live_without_length` ("+N live" beside the sum), and the duration
filter, sort and Home budget never match it. Migration 0117 moves the duration sort indexes onto
the length expression. An AST test holds who may still read the raw column.
The migration 0110 test now ends its read before the way down: 0117 builds an index
CONCURRENTLY there, which waited for that open transaction forever.
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
"""videos: the duration sorts ride an index on the video's length, not on the raw column
|
||||
|
||||
Revision ID: 0117_videos_length_sort_indexes
|
||||
Revises: 0116_followed_explored_channels
|
||||
Create Date: 2026-11-07
|
||||
|
||||
A YouTube length past a day is not a length (`catalog.models.LENGTH_MAX_S`): a 24/7 stream's archive
|
||||
reports the years it has run. The feed's duration sorts now order by `Video.length_seconds`, which
|
||||
is NULL for such a video, so "longest first" no longer opens on a stream archive. Migration 0096's
|
||||
two duration indexes are on the raw column and no longer match that ORDER BY; they are replaced by
|
||||
the same indexes over the length expression. The expression is spelled out here as SQLAlchemy emits
|
||||
it, with the limit as a literal (an index matches a query's constant, never a bound parameter);
|
||||
tests/test_feed_sort_indexes.py fails if the two drift apart.
|
||||
|
||||
CONCURRENTLY (autocommit_block), re-runnable, as in 0096.
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
revision: str = "0117_videos_length_sort_indexes"
|
||||
down_revision: str | None = "0116_followed_explored_channels"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
# Frozen with this migration: `catalog.models.LENGTH_MAX_S` is 24 h.
|
||||
LENGTH = "(CASE WHEN (duration_seconds > 86400) THEN NULL ELSE duration_seconds END)"
|
||||
|
||||
NEW = {
|
||||
"ix_videos_length_desc": f"{LENGTH} DESC NULLS LAST, id",
|
||||
"ix_videos_length_asc": f"{LENGTH} ASC NULLS LAST, id",
|
||||
}
|
||||
OLD = {
|
||||
"ix_videos_duration_desc": "duration_seconds DESC NULLS LAST, id",
|
||||
"ix_videos_duration_asc": "duration_seconds ASC NULLS LAST, id",
|
||||
}
|
||||
|
||||
|
||||
def _valid(name: str) -> bool | None:
|
||||
"""The index's `indisvalid`, or None when there is no index of that name."""
|
||||
row = (
|
||||
op.get_bind()
|
||||
.execute(
|
||||
text(
|
||||
"SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid "
|
||||
"WHERE c.relname = :name"
|
||||
),
|
||||
{"name": name},
|
||||
)
|
||||
.first()
|
||||
)
|
||||
return None if row is None else bool(row[0])
|
||||
|
||||
|
||||
def _build(name: str, columns: str) -> None:
|
||||
"""CREATE INDEX CONCURRENTLY, re-runnable: an INVALID leftover of a failed build is dropped first
|
||||
(see 0096 `_build`)."""
|
||||
if _valid(name) is False:
|
||||
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||
op.execute(f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} ON videos ({columns})")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.get_context().autocommit_block():
|
||||
for name, columns in NEW.items():
|
||||
_build(name, columns)
|
||||
for name in OLD:
|
||||
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.get_context().autocommit_block():
|
||||
for name, columns in OLD.items():
|
||||
_build(name, columns)
|
||||
for name in NEW:
|
||||
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||
@@ -16,10 +16,16 @@ from sqlalchemy import (
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
and_,
|
||||
case,
|
||||
func,
|
||||
literal_column,
|
||||
null,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import TSVECTOR
|
||||
from sqlalchemy.ext.hybrid import hybrid_property
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from app.db import Base
|
||||
from app.models import TimestampMixin
|
||||
@@ -114,6 +120,20 @@ class Subscription(Base, TimestampMixin):
|
||||
# hidden from feeds/search and revisited by the refresh job until they settle into a VOD.
|
||||
LIVE_OR_UPCOMING = ("live", "upcoming")
|
||||
|
||||
# A YouTube length past a day is not a length. A 24/7 stream's archive (`was_live`) reports the years
|
||||
# the stream has run (measured on prod, 2026-11-06: 122,962,914 s), and one such video made a Watch
|
||||
# later total read 44,073 hours. YouTube caps an upload at 12 hours, so nothing real is lost. Such a
|
||||
# length is unknown wherever a length is used: the API sends none (`Video.length_seconds`), sums leave
|
||||
# it out and count it as `live_without_length`, and the duration filter, sort and Home budget never
|
||||
# match it. The raw `duration_seconds` stays for what it does prove — that the video is not a Short —
|
||||
# and for the jobs that ask whether YouTube has answered yet (test_video_length holds who reads it).
|
||||
LENGTH_MAX_S = 24 * 3600
|
||||
|
||||
|
||||
def known_length(seconds: int | None) -> int | None:
|
||||
"""`seconds` as a video's length, or None when it is not one (see LENGTH_MAX_S)."""
|
||||
return seconds if seconds is not None and seconds <= LENGTH_MAX_S else None
|
||||
|
||||
|
||||
class Video(Base, TimestampMixin):
|
||||
"""A YouTube video. Shared across users; per-user state lives elsewhere."""
|
||||
@@ -141,8 +161,8 @@ class Video(Base, TimestampMixin):
|
||||
)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
|
||||
# ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to
|
||||
# `feedquery.order_by` by test_feed_sort_indexes.
|
||||
# ORDER BY, one per direction (and per ICU collation for title): migration 0096 (the duration
|
||||
# pair since 0117, on `length_seconds`), held to `feedquery.order_by` by test_feed_sort_indexes.
|
||||
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
@@ -198,6 +218,33 @@ class Video(Base, TimestampMixin):
|
||||
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
|
||||
|
||||
@hybrid_property
|
||||
def length_seconds(self) -> int | None:
|
||||
"""The video's length, or None when it is unknown — not fetched yet, or not a length at all
|
||||
(`LENGTH_MAX_S`). What every reader of a length reads."""
|
||||
return known_length(self.duration_seconds)
|
||||
|
||||
@length_seconds.inplace.expression
|
||||
@classmethod
|
||||
def _length_seconds_expression(cls) -> ColumnElement[int | None]:
|
||||
# The limit is a literal, not a bound parameter: the duration sort indexes (migration 0117)
|
||||
# are built on this very expression, and Postgres matches an index to a query's constant only.
|
||||
return case(
|
||||
(cls.duration_seconds > literal_column(str(LENGTH_MAX_S)), null()),
|
||||
else_=cls.duration_seconds,
|
||||
)
|
||||
|
||||
@hybrid_property
|
||||
def live_without_length(self) -> bool:
|
||||
"""A live video (on air, upcoming, a premiere or an archive) whose length is unknown: what a
|
||||
total leaves out and names as "+N live"."""
|
||||
return self.live_status != "none" and self.length_seconds is None
|
||||
|
||||
@live_without_length.inplace.expression
|
||||
@classmethod
|
||||
def _live_without_length_expression(cls) -> ColumnElement[bool]:
|
||||
return and_(cls.live_status != "none", cls.length_seconds.is_(None))
|
||||
|
||||
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
|
||||
# creator keywords + search queries (B) > truncated description (C). ts_rank honours the
|
||||
# weights so title matches outrank description matches. Backed by a GIN index (migration
|
||||
|
||||
@@ -44,7 +44,8 @@ def channel_titles(db: Session, ids: set[str]) -> dict[str, str]:
|
||||
|
||||
|
||||
def channel_aggregates(channel_ids: list[str]) -> Select:
|
||||
"""Per-channel stored count, last upload and total duration, in one grouped pass. Every column it
|
||||
"""Per-channel stored count, last upload, total duration and the live videos the total cannot
|
||||
hold (`Video.live_without_length`), in one grouped pass. Every column it
|
||||
reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index
|
||||
alone; a test holds that (test_channel_list_index_only)."""
|
||||
return (
|
||||
@@ -52,15 +53,19 @@ def channel_aggregates(channel_ids: list[str]) -> Select:
|
||||
Video.channel_id,
|
||||
func.count(Video.id),
|
||||
func.max(Video.published_at),
|
||||
func.coalesce(func.sum(Video.duration_seconds), 0),
|
||||
func.coalesce(func.sum(Video.length_seconds), 0),
|
||||
func.count().filter(Video.live_without_length),
|
||||
)
|
||||
.where(Video.channel_id.in_(channel_ids))
|
||||
.group_by(Video.channel_id)
|
||||
)
|
||||
|
||||
|
||||
def aggregates(db: Session, channel_ids: list[str]) -> list[tuple[str, int, datetime | None, int]]:
|
||||
"""`channel_aggregates`, run: (channel id, stored videos, last upload, total seconds)."""
|
||||
def aggregates(
|
||||
db: Session, channel_ids: list[str]
|
||||
) -> list[tuple[str, int, datetime | None, int, int]]:
|
||||
"""`channel_aggregates`, run: (channel id, stored videos, last upload, total seconds, live
|
||||
videos without a length)."""
|
||||
if not channel_ids:
|
||||
return []
|
||||
return list(db.execute(channel_aggregates(channel_ids)).tuples())
|
||||
|
||||
@@ -153,7 +153,7 @@ def populate_from_catalog(db: Session, asset: MediaAsset) -> bool:
|
||||
return False
|
||||
asset.title = video.title
|
||||
asset.thumbnail_url = video.thumbnail_url
|
||||
asset.duration_s = video.duration_seconds
|
||||
asset.duration_s = video.length_seconds
|
||||
if video.published_at:
|
||||
asset.upload_date = video.published_at.date()
|
||||
channel = db.get(Channel, video.channel_id)
|
||||
@@ -337,7 +337,7 @@ def _clip_source_facts(db: Session, spec: dict) -> dict:
|
||||
"title": video.title,
|
||||
"uploader": channel.title if channel else None,
|
||||
"thumbnail_url": video.thumbnail_url,
|
||||
"length_s": video.duration_seconds,
|
||||
"length_s": video.length_seconds,
|
||||
}
|
||||
item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"])
|
||||
if item is None:
|
||||
|
||||
@@ -68,7 +68,7 @@ def feed_columns(user: User, state) -> list:
|
||||
Channel.handle.label("channel_handle"),
|
||||
Video.published_at,
|
||||
Video.thumbnail_url,
|
||||
Video.duration_seconds,
|
||||
Video.length_seconds.label("length_seconds"),
|
||||
Video.view_count,
|
||||
Video.is_short,
|
||||
Video.live_status,
|
||||
@@ -88,7 +88,7 @@ def serialize(row) -> dict:
|
||||
"channel_url": _channel_url(row.channel_id, row.channel_handle),
|
||||
"published_at": iso(row.published_at),
|
||||
"thumbnail_url": row.thumbnail_url,
|
||||
"duration_seconds": row.duration_seconds,
|
||||
"duration_seconds": row.length_seconds,
|
||||
"view_count": row.view_count,
|
||||
"is_short": row.is_short,
|
||||
"live_status": row.live_status,
|
||||
@@ -260,9 +260,9 @@ def filtered_query(
|
||||
# OR across the picked channels; every other filter still applies on top.
|
||||
query = query.where(Video.channel_id.in_(channel_ids))
|
||||
if min_duration is not None:
|
||||
query = query.where(Video.duration_seconds >= min_duration)
|
||||
query = query.where(Video.length_seconds >= min_duration)
|
||||
if max_duration is not None:
|
||||
query = query.where(Video.duration_seconds <= max_duration)
|
||||
query = query.where(Video.length_seconds <= max_duration)
|
||||
if max_age_days is not None:
|
||||
cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400
|
||||
query = query.where(Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc))
|
||||
@@ -503,10 +503,10 @@ def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) ->
|
||||
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
|
||||
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
|
||||
"duration_desc": [
|
||||
{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}
|
||||
{"expr": Video.length_seconds, "asc": False, "nullable": True, "kind": "num"}
|
||||
],
|
||||
"duration_asc": [
|
||||
{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}
|
||||
{"expr": Video.length_seconds, "asc": True, "nullable": True, "kind": "num"}
|
||||
],
|
||||
"title": [{"expr": title, "asc": True, "nullable": True, "kind": "str"}],
|
||||
"title_desc": [{"expr": title, "asc": False, "nullable": True, "kind": "str"}],
|
||||
|
||||
@@ -146,12 +146,17 @@ def item_counts(db: Session, ids: list[int]) -> dict[int, int]:
|
||||
)
|
||||
|
||||
|
||||
def durations(db: Session, ids: list[int]) -> dict[int, int]:
|
||||
"""Each playlist's running time: the sum of its videos' lengths."""
|
||||
def durations(db: Session, ids: list[int]) -> dict[int, tuple[int, int]]:
|
||||
"""Each playlist's running time — the sum of its videos' known lengths — and how many live videos
|
||||
it cannot hold (`Video.live_without_length`)."""
|
||||
return {
|
||||
pid: int(total or 0)
|
||||
for pid, total in db.execute(
|
||||
select(PlaylistItem.playlist_id, func.coalesce(func.sum(Video.duration_seconds), 0))
|
||||
pid: (int(total or 0), int(live or 0))
|
||||
for pid, total, live in db.execute(
|
||||
select(
|
||||
PlaylistItem.playlist_id,
|
||||
func.coalesce(func.sum(Video.length_seconds), 0),
|
||||
func.count().filter(Video.live_without_length),
|
||||
)
|
||||
.join(Video, Video.id == PlaylistItem.video_id)
|
||||
.where(PlaylistItem.playlist_id.in_(ids))
|
||||
.group_by(PlaylistItem.playlist_id)
|
||||
@@ -191,11 +196,16 @@ def covers(db: Session, ids: list[int]) -> dict[int, str | None]:
|
||||
)
|
||||
|
||||
|
||||
def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None]:
|
||||
"""One playlist's item count, running time and cover. An outer join, so an empty list still
|
||||
answers (0, 0, None)."""
|
||||
count, total = db.execute(
|
||||
select(func.count(PlaylistItem.id), func.coalesce(func.sum(Video.duration_seconds), 0))
|
||||
def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, int, str | None]:
|
||||
"""One playlist's item count, running time, live videos the running time cannot hold
|
||||
(`Video.live_without_length`) and cover. An outer join, so an empty list still answers
|
||||
(0, 0, 0, None)."""
|
||||
count, total, live = db.execute(
|
||||
select(
|
||||
func.count(PlaylistItem.id),
|
||||
func.coalesce(func.sum(Video.length_seconds), 0),
|
||||
func.count().filter(Video.live_without_length),
|
||||
)
|
||||
.select_from(PlaylistItem)
|
||||
.outerjoin(Video, Video.id == PlaylistItem.video_id)
|
||||
.where(PlaylistItem.playlist_id == playlist_id)
|
||||
@@ -207,7 +217,7 @@ def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None
|
||||
.order_by(PlaylistItem.position)
|
||||
.limit(1)
|
||||
)
|
||||
return count or 0, total or 0, cover
|
||||
return count or 0, total or 0, live or 0, cover
|
||||
|
||||
|
||||
def playlist_rows(db: Session, user: User, playlist_id: int) -> list:
|
||||
|
||||
@@ -46,11 +46,12 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_
|
||||
# computed here — the manager table dropped its N/S/L column.)
|
||||
channel_ids = [c.id for _, c in rows]
|
||||
agg: dict[str, dict] = {}
|
||||
for cid, n, last_up, total_dur in queries.aggregates(db, channel_ids):
|
||||
for cid, n, last_up, total_dur, live in queries.aggregates(db, channel_ids):
|
||||
agg[cid] = {
|
||||
"stored": n,
|
||||
"last_video_at": iso(last_up),
|
||||
"total_duration_seconds": int(total_dur or 0),
|
||||
"live_without_length": live,
|
||||
}
|
||||
|
||||
# Channels already in the *global* deep-backfill queue — i.e. at least one user (any
|
||||
@@ -68,6 +69,7 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_
|
||||
"stored_videos": (agg.get(ch.id) or {}).get("stored", 0),
|
||||
"last_video_at": (agg.get(ch.id) or {}).get("last_video_at"),
|
||||
"total_duration_seconds": (agg.get(ch.id) or {}).get("total_duration_seconds", 0),
|
||||
"live_without_length": (agg.get(ch.id) or {}).get("live_without_length", 0),
|
||||
"priority": sub.priority,
|
||||
"hidden": sub.hidden,
|
||||
"deep_requested": sub.deep_requested,
|
||||
|
||||
@@ -10,6 +10,7 @@ from sqlalchemy.orm.exc import StaleDataError
|
||||
|
||||
from app import jobruns, quota, scheduler
|
||||
from app.catalog import queries as catalog_queries, service as catalog_service
|
||||
from app.catalog.models import known_length
|
||||
from app.auth import current_user
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.mailtext import lang_for_user
|
||||
@@ -287,7 +288,7 @@ def get_video_detail(
|
||||
"channel_title": v.channel.title if v.channel else None,
|
||||
"published_at": iso(v.published_at),
|
||||
"view_count": v.view_count,
|
||||
"duration_seconds": v.duration_seconds,
|
||||
"duration_seconds": v.length_seconds,
|
||||
}
|
||||
|
||||
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit.
|
||||
@@ -315,8 +316,8 @@ def get_video_detail(
|
||||
"channel_title": snippet.get("channelTitle"),
|
||||
"published_at": snippet.get("publishedAt"),
|
||||
"view_count": int(views) if views is not None else None,
|
||||
"duration_seconds": parse_iso8601_duration(
|
||||
items[0].get("contentDetails", {}).get("duration")
|
||||
"duration_seconds": known_length(
|
||||
parse_iso8601_duration(items[0].get("contentDetails", {}).get("duration"))
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -95,10 +95,13 @@ def _video_or_404(db: Session, video_id: str) -> None:
|
||||
def _summary(
|
||||
pl: "Playlist",
|
||||
count: int,
|
||||
total_duration: int = 0,
|
||||
duration: tuple[int, int] = (0, 0),
|
||||
has_video: bool | None = None,
|
||||
cover: str | None = None,
|
||||
) -> dict:
|
||||
"""`duration` is `queries.durations`'s pair: the known running time, and the live videos it
|
||||
cannot hold — the "+N live" beside it."""
|
||||
total_duration, live = duration
|
||||
out = {
|
||||
"id": pl.id,
|
||||
"name": pl.name,
|
||||
@@ -108,6 +111,7 @@ def _summary(
|
||||
"dirty": pl.dirty,
|
||||
"item_count": count,
|
||||
"total_duration_seconds": total_duration,
|
||||
"live_without_length": live,
|
||||
"cover_thumbnail": cover,
|
||||
# When this list last CHANGED (rename, add/remove/move/reorder) — the rail's
|
||||
# "recently changed first" sort reads it. See `service.touch` for why it isn't automatic.
|
||||
@@ -138,7 +142,7 @@ def list_playlists(
|
||||
_summary(
|
||||
p,
|
||||
counts.get(p.id, 0),
|
||||
durations.get(p.id, 0),
|
||||
durations.get(p.id, (0, 0)),
|
||||
(p.id in member) if contains else None,
|
||||
covers.get(p.id),
|
||||
)
|
||||
@@ -363,8 +367,8 @@ def rename_playlist(
|
||||
raise _refused(400, "name_required")
|
||||
service.rename_playlist(db, pl, name)
|
||||
db.commit()
|
||||
count, total, cover = queries.playlist_totals(db, pl.id)
|
||||
return _summary(pl, count, total, cover=cover)
|
||||
count, total, live, cover = queries.playlist_totals(db, pl.id)
|
||||
return _summary(pl, count, (total, live), cover=cover)
|
||||
|
||||
|
||||
@router.delete("/{playlist_id}")
|
||||
|
||||
@@ -45,6 +45,9 @@ def test_an_interval_override_moves_into_sysconfig_and_back(db):
|
||||
}
|
||||
assert conn.execute(text("SELECT to_regclass('scheduler_settings')")).scalar() is None
|
||||
assert scheduler.load_intervals(db)["rss_poll"] == 25
|
||||
# The way down rebuilds an index CONCURRENTLY (0117), which waits for every open transaction —
|
||||
# the read above left this session's open.
|
||||
db.rollback()
|
||||
|
||||
command.downgrade(cfg, "0109_job_runs")
|
||||
try:
|
||||
|
||||
@@ -252,7 +252,7 @@ def test_an_add_is_idempotent_and_lands_where_asked(db, make_user):
|
||||
db.commit()
|
||||
assert _ids(db, pl) == ["c", "b"]
|
||||
assert queries.item_counts(db, [pl.id]) == {pl.id: 2}
|
||||
assert queries.durations(db, [pl.id]) == {pl.id: 120}
|
||||
assert queries.durations(db, [pl.id]) == {pl.id: (120, 0)}
|
||||
assert queries.containing(db, [pl.id], "b") == {pl.id}
|
||||
|
||||
|
||||
@@ -294,7 +294,7 @@ def test_a_rename_and_a_delete(db, make_user):
|
||||
service.add_item(db, pl, "a", mark_dirty=False)
|
||||
service.rename_playlist(db, pl, "New")
|
||||
db.commit()
|
||||
assert queries.playlist_totals(db, pl.id) == (1, 60, None)
|
||||
assert queries.playlist_totals(db, pl.id) == (1, 60, 0, None)
|
||||
assert pl.name == "New"
|
||||
service.delete_playlist(db, pl)
|
||||
db.commit()
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
"""A YouTube length past a day is unknown, everywhere a length is used (`catalog.models.LENGTH_MAX_S`).
|
||||
|
||||
Measured on prod (2026-11-06): two 24/7 streams' archives report 122,962,914 s and 34,762,459 s, and the
|
||||
owner's Watch later total read "44,073 hr 55 min". Such a video now has no length on any API answer, the
|
||||
totals leave it out and count it as `live_without_length`, and the duration filter and sort pass it over
|
||||
like a video whose length is not fetched yet. An AST check holds who may still read the raw column.
|
||||
DB lane (the AST check needs none).
|
||||
"""
|
||||
|
||||
import ast
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
from app.catalog.models import LENGTH_MAX_S, Channel, Subscription, Video, known_length
|
||||
from app.reader.models import Playlist, PlaylistItem
|
||||
|
||||
YEARS = 122_962_914 # the measured archive
|
||||
HOUR = 3600
|
||||
|
||||
|
||||
def _seed(db, user):
|
||||
db.add(Channel(id="UClen", title="Len"))
|
||||
db.flush()
|
||||
db.add(Subscription(user_id=user.id, channel_id="UClen", priority=0, hidden=False))
|
||||
rows = [
|
||||
("short", 600, "none"),
|
||||
("long", 2 * HOUR, "none"),
|
||||
("day", LENGTH_MAX_S, "was_live"), # exactly a day still is a length
|
||||
("archive", YEARS, "was_live"),
|
||||
("onair", None, "live"),
|
||||
("unfetched", None, "none"), # not live: not counted as "+N live"
|
||||
]
|
||||
for i, (vid, seconds, live) in enumerate(rows):
|
||||
db.add(
|
||||
Video(
|
||||
id=vid,
|
||||
channel_id="UClen",
|
||||
published_at=datetime(2024, 1, i + 1, tzinfo=timezone.utc),
|
||||
title=vid,
|
||||
duration_seconds=seconds,
|
||||
live_status=live,
|
||||
is_short=False,
|
||||
via_search=False,
|
||||
via_explore=False,
|
||||
)
|
||||
)
|
||||
pl = Playlist(user_id=user.id, name="Later")
|
||||
db.add(pl)
|
||||
db.flush()
|
||||
for i, (vid, _s, _l) in enumerate(rows):
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=i))
|
||||
db.commit()
|
||||
return pl
|
||||
|
||||
|
||||
KNOWN_SUM = 600 + 2 * HOUR + LENGTH_MAX_S
|
||||
|
||||
|
||||
def test_the_rule():
|
||||
assert known_length(None) is None
|
||||
assert known_length(LENGTH_MAX_S) == LENGTH_MAX_S
|
||||
assert known_length(LENGTH_MAX_S + 1) is None
|
||||
assert known_length(YEARS) is None
|
||||
|
||||
|
||||
def test_the_sql_length_carries_its_limit_as_a_literal():
|
||||
# The duration sort indexes (migration 0117) are built on this expression with 86400 in it. A
|
||||
# bound parameter matches no index once the driver reuses a prepared, generic plan, and the
|
||||
# sort-index test cannot see that: it compiles with literal binds.
|
||||
compiled = Video.length_seconds.expression.compile(dialect=postgresql.dialect())
|
||||
assert compiled.params == {}
|
||||
assert f"> {LENGTH_MAX_S}" in str(compiled)
|
||||
|
||||
|
||||
def test_a_years_long_archive_has_no_length_in_the_feed_or_a_playlist(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user)
|
||||
client = as_user(user)
|
||||
|
||||
feed = client.get("/api/feed?show=all&include_live=true&limit=50")
|
||||
assert feed.status_code == 200
|
||||
by_id = {v["id"]: v["duration_seconds"] for v in feed.json()["items"]}
|
||||
assert by_id["archive"] is None
|
||||
assert by_id["day"] == LENGTH_MAX_S
|
||||
|
||||
detail = client.get(f"/api/playlists/{pl.id}")
|
||||
assert detail.status_code == 200
|
||||
items = {v["id"]: v["duration_seconds"] for v in detail.json()["items"]}
|
||||
assert items["archive"] is None
|
||||
|
||||
one = client.get("/api/videos/archive")
|
||||
assert one.status_code == 200
|
||||
assert one.json()["duration_seconds"] is None
|
||||
|
||||
|
||||
def test_the_totals_leave_it_out_and_count_the_live_ones(db, make_user, as_user):
|
||||
user = make_user()
|
||||
pl = _seed(db, user)
|
||||
client = as_user(user)
|
||||
|
||||
listed = client.get("/api/playlists")
|
||||
assert listed.status_code == 200
|
||||
(summary,) = [p for p in listed.json() if p["id"] == pl.id]
|
||||
assert summary["total_duration_seconds"] == KNOWN_SUM
|
||||
assert summary["live_without_length"] == 2 # the archive and the one on air
|
||||
|
||||
renamed = client.patch(f"/api/playlists/{pl.id}", json={"name": "Later 2"})
|
||||
assert renamed.status_code == 200
|
||||
assert renamed.json()["total_duration_seconds"] == KNOWN_SUM
|
||||
assert renamed.json()["live_without_length"] == 2
|
||||
|
||||
channels = client.get("/api/channels")
|
||||
assert channels.status_code == 200
|
||||
(ch,) = channels.json()
|
||||
assert ch["total_duration_seconds"] == KNOWN_SUM
|
||||
assert ch["live_without_length"] == 2
|
||||
|
||||
|
||||
def test_the_duration_filter_and_sort_pass_it_over(db, make_user, as_user):
|
||||
user = make_user()
|
||||
_seed(db, user)
|
||||
client = as_user(user)
|
||||
base = "/api/feed?show=all&include_live=true&limit=50"
|
||||
|
||||
longer = client.get(f"{base}&min_duration={HOUR}")
|
||||
assert longer.status_code == 200
|
||||
assert {v["id"] for v in longer.json()["items"]} == {"long", "day"}
|
||||
|
||||
longest = client.get(f"{base}&sort=duration_desc")
|
||||
assert longest.status_code == 200
|
||||
order = [v["id"] for v in longest.json()["items"]]
|
||||
assert order[:3] == ["day", "long", "short"]
|
||||
assert set(order[3:]) == {"archive", "onair", "unfetched"}
|
||||
|
||||
|
||||
# Who may read `duration_seconds` itself, by (file, function): what each one asks is not "how long".
|
||||
RAW_READERS = {
|
||||
# The column is defined and turned into a length here.
|
||||
("catalog/models.py", "length_seconds"),
|
||||
("catalog/models.py", "_length_seconds_expression"),
|
||||
# Ingest writes it; the live refresh asks whether YouTube has answered for an archive yet; the
|
||||
# Shorts classification needs only "longer than a Short", which a years-long value proves.
|
||||
("sync/videos.py", "apply_video_details"),
|
||||
("sync/videos.py", "refresh_live"),
|
||||
("sync/videos.py", "run_shorts_classification"),
|
||||
("catalog/search.py", "classify_shorts"),
|
||||
("catalog/queries.py", "work_queue"),
|
||||
# Not a video's column: the player's own report, and the watch history's looked-up lengths, which
|
||||
# count a watched video as more than a glance (a watched archive is that too).
|
||||
("routes/_common.py", "kept_position"),
|
||||
("routes/plex/playback.py", "item_progress"),
|
||||
("reader/queries.py", "_counted_channels"),
|
||||
("reader/queries.py", "watched_channels"),
|
||||
("reader/queries.py", "watched_channel_videos"),
|
||||
}
|
||||
|
||||
|
||||
def _raw_reads() -> set[tuple[str, str]]:
|
||||
root = Path(__file__).resolve().parents[1] / "app"
|
||||
found: set[tuple[str, str]] = set()
|
||||
for path in root.rglob("*.py"):
|
||||
if "alembic" in path.parts:
|
||||
continue
|
||||
rel = path.relative_to(root).as_posix()
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
for fn in ast.walk(tree):
|
||||
if not isinstance(fn, ast.FunctionDef):
|
||||
continue
|
||||
for node in ast.walk(fn):
|
||||
if isinstance(node, ast.Attribute) and node.attr == "duration_seconds":
|
||||
found.add((rel, fn.name))
|
||||
return found
|
||||
|
||||
|
||||
def test_only_the_listed_readers_take_the_raw_column():
|
||||
found = _raw_reads()
|
||||
stray = sorted(found - RAW_READERS)
|
||||
assert not stray, f"read `length_seconds` (a length) instead of the raw column: {stray}"
|
||||
# A listed reader that stopped reading it leaves the list.
|
||||
assert not RAW_READERS - found, sorted(RAW_READERS - found)
|
||||
@@ -0,0 +1,21 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { TFunction } from "i18next";
|
||||
import { isLiveWithoutLength, withLiveCount } from "./liveWithoutLength";
|
||||
|
||||
const t = ((key: string, o: { count: number }) => `${key}:${o.count}`) as unknown as TFunction;
|
||||
|
||||
describe("a total and the live videos it cannot hold", () => {
|
||||
it("counts a live video only when the server sent no length", () => {
|
||||
expect(isLiveWithoutLength({ duration_seconds: null, live_status: "was_live" })).toBe(true);
|
||||
expect(isLiveWithoutLength({ duration_seconds: null, live_status: "live" })).toBe(true);
|
||||
expect(isLiveWithoutLength({ duration_seconds: 5400, live_status: "was_live" })).toBe(false);
|
||||
// Not fetched yet, and not live: left out of the sum like before, not named as live.
|
||||
expect(isLiveWithoutLength({ duration_seconds: null, live_status: "none" })).toBe(false);
|
||||
});
|
||||
|
||||
it("names them beside the sum, or alone when nothing summed", () => {
|
||||
expect(withLiveCount("12 hr 5 min", 2, t)).toBe("12 hr 5 min card.liveWithoutLength:2");
|
||||
expect(withLiveCount("", 1, t)).toBe("card.liveWithoutLength:1");
|
||||
expect(withLiveCount("12 hr 5 min", 0, t)).toBe("12 hr 5 min");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
// A total of video lengths and the live videos it cannot hold. The server decides what a length is
|
||||
// (a YouTube "length" past a day is a 24/7 stream's archive reporting the years it has run, and the
|
||||
// API sends none for it: `catalog.models.LENGTH_MAX_S`), so this side keeps no copy of that limit. It
|
||||
// only names what a sum left out: a live video — on air, upcoming, a premiere or an archive — with no
|
||||
// length, as the server's `Video.live_without_length` counts it for the totals it sums itself.
|
||||
import type { TFunction } from "i18next";
|
||||
import type { Video } from "../../lib/api/feed";
|
||||
|
||||
export function isLiveWithoutLength(v: Pick<Video, "duration_seconds" | "live_status">): boolean {
|
||||
return v.duration_seconds == null && v.live_status !== "none";
|
||||
}
|
||||
|
||||
/** "12 hr 5 min +2 live": `total` is the formatted sum ("" when nothing summed), `live` the count
|
||||
* it left out. Empty when both are. */
|
||||
export function withLiveCount(total: string, live: number, t: TFunction): string {
|
||||
const suffix = live > 0 ? t("card.liveWithoutLength", { count: live }) : "";
|
||||
return [total, suffix].filter(Boolean).join(" ");
|
||||
}
|
||||
@@ -16,6 +16,8 @@
|
||||
"restart": "Restart",
|
||||
"play": "Play",
|
||||
"stream": "stream",
|
||||
"liveWithoutLength_one": "+{{count}} live",
|
||||
"liveWithoutLength_other": "+{{count}} live",
|
||||
"views": "views",
|
||||
"info": "Details",
|
||||
"noDescription": "No description",
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
"restart": "Elölről",
|
||||
"play": "Lejátszás",
|
||||
"stream": "közvetítés",
|
||||
"liveWithoutLength_one": "+{{count}} élő",
|
||||
"liveWithoutLength_other": "+{{count}} élő",
|
||||
"views": "megtekintés",
|
||||
"info": "Részletek",
|
||||
"noDescription": "Nincs leírás",
|
||||
|
||||
@@ -77,6 +77,8 @@ export interface ManagedChannel {
|
||||
* subscribe made here. */
|
||||
added_at: string;
|
||||
total_duration_seconds: number;
|
||||
/** Live videos the total cannot hold (no length): the "+N live" beside it. */
|
||||
live_without_length: number;
|
||||
priority: number;
|
||||
hidden: boolean;
|
||||
deep_requested: boolean;
|
||||
|
||||
@@ -17,6 +17,8 @@ export interface Playlist {
|
||||
dirty: boolean; // linked local playlist has edits not yet pushed to YouTube
|
||||
item_count: number;
|
||||
total_duration_seconds: number;
|
||||
/** Live videos the total cannot hold (no length): the "+N live" beside it. */
|
||||
live_without_length: number;
|
||||
cover_thumbnail: string | null;
|
||||
updated_at: string; // ISO; when the list last CHANGED (rename, add/remove/move/reorder)
|
||||
has_video?: boolean; // only set when listed with ?contains=<video_id>
|
||||
|
||||
@@ -28,6 +28,7 @@ const ch = (over: Partial<ManagedChannel>): ManagedChannel => ({
|
||||
last_video_at: null,
|
||||
added_at: "2026-06-10T08:00:00+00:00",
|
||||
total_duration_seconds: 0,
|
||||
live_without_length: 0,
|
||||
priority: 0,
|
||||
hidden: false,
|
||||
deep_requested: false,
|
||||
|
||||
@@ -11,6 +11,7 @@ import { ChannelTagsControl, type SuggestedTag } from "../../features/channel/Ch
|
||||
import { StoredCell } from "./channelCells";
|
||||
import { isFullySynced } from "./channelSync";
|
||||
import { RelativeTime } from "../../components/ui/RelativeTime";
|
||||
import { withLiveCount } from "../../features/video/liveWithoutLength";
|
||||
import type { FilterChip } from "./TagFilterChips";
|
||||
|
||||
/** The shared "subscribers" DataTable column (right-aligned, sortable, em-dash for null) — identical
|
||||
@@ -236,6 +237,10 @@ export function buildChannelColumns(deps: ChannelColumnDeps): Column<ManagedChan
|
||||
render: (c) => (
|
||||
<span className="text-muted tabular-nums">
|
||||
{formatTotalHours(c.total_duration_seconds, lang)}
|
||||
{/* Its own line: the column is sized for the sum alone. */}
|
||||
{c.live_without_length > 0 && (
|
||||
<span className="block text-2xs">{withLiveCount("", c.live_without_length, t)}</span>
|
||||
)}
|
||||
</span>
|
||||
),
|
||||
},
|
||||
|
||||
@@ -61,6 +61,7 @@ const ch = (over: Partial<ManagedChannel> = {}): ManagedChannel => ({
|
||||
last_video_at: null,
|
||||
added_at: "2026-06-10T08:00:00+00:00",
|
||||
total_duration_seconds: 0,
|
||||
live_without_length: 0,
|
||||
priority: 0,
|
||||
hidden: false,
|
||||
deep_requested: false,
|
||||
|
||||
@@ -134,6 +134,7 @@ const vid = (id: string, position_seconds: number): Video =>
|
||||
status: "new",
|
||||
position_seconds,
|
||||
duration_seconds: 3000,
|
||||
live_status: "none",
|
||||
published_at: "2026-01-01T00:00:00Z",
|
||||
watch_url: `https://www.youtube.com/watch?v=${id}`,
|
||||
}) as unknown as Video;
|
||||
@@ -998,6 +999,17 @@ describe("Playlists · the header", () => {
|
||||
expect(await meta()).toBe("playlists.itemCount");
|
||||
});
|
||||
|
||||
it("names the live videos the sum cannot hold beside it", async () => {
|
||||
// A 24/7 stream's archive comes with no length (the server's rule): it is named, not summed.
|
||||
const archive = (id: string) => ({ ...untimed(id), live_status: "was_live" }) as Video;
|
||||
mount({ 1: [vid("A", 0), archive("B")] });
|
||||
expect(await meta()).toBe("playlists.itemCount · 50 min card.liveWithoutLength");
|
||||
cleanup();
|
||||
|
||||
mount({ 1: [archive("C")] });
|
||||
expect(await meta()).toBe("playlists.itemCount · card.liveWithoutLength");
|
||||
});
|
||||
|
||||
it("offers the view controls only once there is more than one row to arrange", async () => {
|
||||
mount({ 1: [vid("A", 0)] });
|
||||
await screen.findByText("Video A");
|
||||
|
||||
@@ -68,6 +68,7 @@ import {
|
||||
playlistViewParamUpdates,
|
||||
} from "../../lib/playlistsUrlState";
|
||||
import { formatRuntime } from "../../lib/format";
|
||||
import { isLiveWithoutLength, withLiveCount } from "../../features/video/liveWithoutLength";
|
||||
import { applyView, sameOrder } from "../../lib/playlistView";
|
||||
import { useOpenLinkedPlayer } from "../../features/player/useOpenLinkedPlayer";
|
||||
const PlayerModal = lazy(() => import("../../features/player/PlayerModal"));
|
||||
@@ -565,11 +566,13 @@ function PlaylistDetail() {
|
||||
}, [items]);
|
||||
|
||||
// Total playtime of the whole playlist (sums every item, matching the itemCount beside it — not
|
||||
// the filtered `shown` set). Items with an unknown duration count as 0; hidden when nothing sums.
|
||||
// the filtered `shown` set). Items with an unknown duration count as 0, and a live one is named
|
||||
// beside the sum instead ("+2 live"); hidden when there is neither.
|
||||
const totalSeconds = useMemo(
|
||||
() => items.reduce((acc, v) => acc + (v.duration_seconds ?? 0), 0),
|
||||
[items],
|
||||
);
|
||||
const liveCount = useMemo(() => items.filter(isLiveWithoutLength).length, [items]);
|
||||
|
||||
// The row's "…" opens the row menu under itself. Which row's menu is open is part of the
|
||||
// cell (aria-expanded), so the columns rebuild when it changes — a cheap array of closures.
|
||||
@@ -1619,7 +1622,8 @@ function PlaylistDetail() {
|
||||
>
|
||||
<span className="text-xs text-muted">
|
||||
{t("playlists.itemCount", { count: items.length })}
|
||||
{totalSeconds > 0 && ` · ${formatRuntime(totalSeconds, lang)}`}
|
||||
{(totalSeconds > 0 || liveCount > 0) &&
|
||||
` · ${withLiveCount(formatRuntime(totalSeconds, lang), liveCount, t)}`}
|
||||
{filterActive &&
|
||||
` · ${t("playlists.filteredCount", { count: shown.length })}`}
|
||||
</span>
|
||||
|
||||
@@ -75,6 +75,7 @@ const pl = (id: number, name: string) =>
|
||||
yt_playlist_id: null,
|
||||
item_count: 1,
|
||||
total_duration_seconds: 0,
|
||||
live_without_length: 0,
|
||||
cover_thumbnail: null,
|
||||
dirty: false,
|
||||
}) as unknown as Playlist;
|
||||
|
||||
@@ -80,6 +80,7 @@ const alpha = {
|
||||
source: "local",
|
||||
item_count: 3,
|
||||
total_duration_seconds: 0,
|
||||
live_without_length: 0,
|
||||
cover_thumbnail: null,
|
||||
dirty: false,
|
||||
} as unknown as Playlist;
|
||||
|
||||
@@ -8,6 +8,7 @@ import { YouTubeIcon } from "../../lib/brandIcons";
|
||||
import { api, type Playlist } from "../../lib/api";
|
||||
import { compareText } from "../../lib/collate";
|
||||
import { formatRuntime } from "../../lib/format";
|
||||
import { withLiveCount } from "../../features/video/liveWithoutLength";
|
||||
import { msg, notify } from "../../lib/notifications";
|
||||
import { notifyYouTubeActionError } from "../../lib/youtubeErrors";
|
||||
import { LS } from "../../lib/storage";
|
||||
@@ -378,8 +379,12 @@ export default function PlaylistsRail() {
|
||||
</div>
|
||||
<div className="text-2xs text-muted">
|
||||
{t("playlists.itemCount", { count: pl.item_count })}
|
||||
{pl.total_duration_seconds > 0 &&
|
||||
` · ${formatRuntime(pl.total_duration_seconds, lang)}`}
|
||||
{(pl.total_duration_seconds > 0 || pl.live_without_length > 0) &&
|
||||
` · ${withLiveCount(
|
||||
formatRuntime(pl.total_duration_seconds, lang),
|
||||
pl.live_without_length,
|
||||
t,
|
||||
)}`}
|
||||
</div>
|
||||
</div>
|
||||
</Pressable>
|
||||
|
||||
Reference in New Issue
Block a user