fix(catalog): a 24/7 stream archive's years-long length is unknown, not summed or sorted by

YouTube reports a 24/7 stream's archive with the years it has run (prod: 122,962,914 s), and one
such video made the Watch later total read 44,073 hours. A length past a day is now no length
(`Video.length_seconds`, LENGTH_MAX_S): the API sends none, the playlist and channel totals
leave it out and count it as `live_without_length` ("+N live" beside the sum), and the duration
filter, sort and Home budget never match it. Migration 0117 moves the duration sort indexes onto
the length expression. An AST test holds who may still read the raw column.

The migration 0110 test now ends its read before the way down: 0117 builds an index
CONCURRENTLY there, which waited for that open transaction forever.
This commit is contained in:
2026-11-07 02:20:03 +01:00
parent 1bf3d6a592
commit 5b846136ad
26 changed files with 446 additions and 39 deletions
+49 -2
View File
@@ -16,10 +16,16 @@ from sqlalchemy import (
String,
Text,
UniqueConstraint,
and_,
case,
func,
literal_column,
null,
)
from sqlalchemy.dialects.postgresql import TSVECTOR
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import Mapped, mapped_column, relationship
from sqlalchemy.sql.elements import ColumnElement
from app.db import Base
from app.models import TimestampMixin
@@ -114,6 +120,20 @@ class Subscription(Base, TimestampMixin):
# hidden from feeds/search and revisited by the refresh job until they settle into a VOD.
LIVE_OR_UPCOMING = ("live", "upcoming")
# A YouTube length past a day is not a length. A 24/7 stream's archive (`was_live`) reports the years
# the stream has run (measured on prod, 2026-11-06: 122,962,914 s), and one such video made a Watch
# later total read 44,073 hours. YouTube caps an upload at 12 hours, so nothing real is lost. Such a
# length is unknown wherever a length is used: the API sends none (`Video.length_seconds`), sums leave
# it out and count it as `live_without_length`, and the duration filter, sort and Home budget never
# match it. The raw `duration_seconds` stays for what it does prove — that the video is not a Short —
# and for the jobs that ask whether YouTube has answered yet (test_video_length holds who reads it).
LENGTH_MAX_S = 24 * 3600
def known_length(seconds: int | None) -> int | None:
"""`seconds` as a video's length, or None when it is not one (see LENGTH_MAX_S)."""
return seconds if seconds is not None and seconds <= LENGTH_MAX_S else None
class Video(Base, TimestampMixin):
"""A YouTube video. Shared across users; per-user state lives elsewhere."""
@@ -141,8 +161,8 @@ class Video(Base, TimestampMixin):
)
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
# ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to
# `feedquery.order_by` by test_feed_sort_indexes.
# ORDER BY, one per direction (and per ICU collation for title): migration 0096 (the duration
# pair since 0117, on `length_seconds`), held to `feedquery.order_by` by test_feed_sort_indexes.
duration_seconds: Mapped[int | None] = mapped_column(Integer)
view_count: Mapped[int | None] = mapped_column(BigInteger)
like_count: Mapped[int | None] = mapped_column(BigInteger)
@@ -198,6 +218,33 @@ class Video(Base, TimestampMixin):
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
@hybrid_property
def length_seconds(self) -> int | None:
"""The video's length, or None when it is unknown — not fetched yet, or not a length at all
(`LENGTH_MAX_S`). What every reader of a length reads."""
return known_length(self.duration_seconds)
@length_seconds.inplace.expression
@classmethod
def _length_seconds_expression(cls) -> ColumnElement[int | None]:
# The limit is a literal, not a bound parameter: the duration sort indexes (migration 0117)
# are built on this very expression, and Postgres matches an index to a query's constant only.
return case(
(cls.duration_seconds > literal_column(str(LENGTH_MAX_S)), null()),
else_=cls.duration_seconds,
)
@hybrid_property
def live_without_length(self) -> bool:
"""A live video (on air, upcoming, a premiere or an archive) whose length is unknown: what a
total leaves out and names as "+N live"."""
return self.live_status != "none" and self.length_seconds is None
@live_without_length.inplace.expression
@classmethod
def _live_without_length_expression(cls) -> ColumnElement[bool]:
return and_(cls.live_status != "none", cls.length_seconds.is_(None))
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
# creator keywords + search queries (B) > truncated description (C). ts_rank honours the
# weights so title matches outrank description matches. Backed by a GIN index (migration
+9 -4
View File
@@ -44,7 +44,8 @@ def channel_titles(db: Session, ids: set[str]) -> dict[str, str]:
def channel_aggregates(channel_ids: list[str]) -> Select:
"""Per-channel stored count, last upload and total duration, in one grouped pass. Every column it
"""Per-channel stored count, last upload, total duration and the live videos the total cannot
hold (`Video.live_without_length`), in one grouped pass. Every column it
reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index
alone; a test holds that (test_channel_list_index_only)."""
return (
@@ -52,15 +53,19 @@ def channel_aggregates(channel_ids: list[str]) -> Select:
Video.channel_id,
func.count(Video.id),
func.max(Video.published_at),
func.coalesce(func.sum(Video.duration_seconds), 0),
func.coalesce(func.sum(Video.length_seconds), 0),
func.count().filter(Video.live_without_length),
)
.where(Video.channel_id.in_(channel_ids))
.group_by(Video.channel_id)
)
def aggregates(db: Session, channel_ids: list[str]) -> list[tuple[str, int, datetime | None, int]]:
"""`channel_aggregates`, run: (channel id, stored videos, last upload, total seconds)."""
def aggregates(
db: Session, channel_ids: list[str]
) -> list[tuple[str, int, datetime | None, int, int]]:
"""`channel_aggregates`, run: (channel id, stored videos, last upload, total seconds, live
videos without a length)."""
if not channel_ids:
return []
return list(db.execute(channel_aggregates(channel_ids)).tuples())
+2 -2
View File
@@ -153,7 +153,7 @@ def populate_from_catalog(db: Session, asset: MediaAsset) -> bool:
return False
asset.title = video.title
asset.thumbnail_url = video.thumbnail_url
asset.duration_s = video.duration_seconds
asset.duration_s = video.length_seconds
if video.published_at:
asset.upload_date = video.published_at.date()
channel = db.get(Channel, video.channel_id)
@@ -337,7 +337,7 @@ def _clip_source_facts(db: Session, spec: dict) -> dict:
"title": video.title,
"uploader": channel.title if channel else None,
"thumbnail_url": video.thumbnail_url,
"length_s": video.duration_seconds,
"length_s": video.length_seconds,
}
item = plex_lookups.by_rating_key(db, PlexItem, spec["ref"])
if item is None:
+6 -6
View File
@@ -68,7 +68,7 @@ def feed_columns(user: User, state) -> list:
Channel.handle.label("channel_handle"),
Video.published_at,
Video.thumbnail_url,
Video.duration_seconds,
Video.length_seconds.label("length_seconds"),
Video.view_count,
Video.is_short,
Video.live_status,
@@ -88,7 +88,7 @@ def serialize(row) -> dict:
"channel_url": _channel_url(row.channel_id, row.channel_handle),
"published_at": iso(row.published_at),
"thumbnail_url": row.thumbnail_url,
"duration_seconds": row.duration_seconds,
"duration_seconds": row.length_seconds,
"view_count": row.view_count,
"is_short": row.is_short,
"live_status": row.live_status,
@@ -260,9 +260,9 @@ def filtered_query(
# OR across the picked channels; every other filter still applies on top.
query = query.where(Video.channel_id.in_(channel_ids))
if min_duration is not None:
query = query.where(Video.duration_seconds >= min_duration)
query = query.where(Video.length_seconds >= min_duration)
if max_duration is not None:
query = query.where(Video.duration_seconds <= max_duration)
query = query.where(Video.length_seconds <= max_duration)
if max_age_days is not None:
cutoff = datetime.now(timezone.utc).timestamp() - max_age_days * 86400
query = query.where(Video.published_at >= datetime.fromtimestamp(cutoff, tz=timezone.utc))
@@ -503,10 +503,10 @@ def sort_keys(sort: str, seed: int, rank_expr=None, lang: str | None = None) ->
"views": [{"expr": Video.view_count, "asc": False, "nullable": True, "kind": "num"}],
"views_asc": [{"expr": Video.view_count, "asc": True, "nullable": True, "kind": "num"}],
"duration_desc": [
{"expr": Video.duration_seconds, "asc": False, "nullable": True, "kind": "num"}
{"expr": Video.length_seconds, "asc": False, "nullable": True, "kind": "num"}
],
"duration_asc": [
{"expr": Video.duration_seconds, "asc": True, "nullable": True, "kind": "num"}
{"expr": Video.length_seconds, "asc": True, "nullable": True, "kind": "num"}
],
"title": [{"expr": title, "asc": True, "nullable": True, "kind": "str"}],
"title_desc": [{"expr": title, "asc": False, "nullable": True, "kind": "str"}],
+21 -11
View File
@@ -146,12 +146,17 @@ def item_counts(db: Session, ids: list[int]) -> dict[int, int]:
)
def durations(db: Session, ids: list[int]) -> dict[int, int]:
"""Each playlist's running time: the sum of its videos' lengths."""
def durations(db: Session, ids: list[int]) -> dict[int, tuple[int, int]]:
"""Each playlist's running time — the sum of its videos' known lengths — and how many live videos
it cannot hold (`Video.live_without_length`)."""
return {
pid: int(total or 0)
for pid, total in db.execute(
select(PlaylistItem.playlist_id, func.coalesce(func.sum(Video.duration_seconds), 0))
pid: (int(total or 0), int(live or 0))
for pid, total, live in db.execute(
select(
PlaylistItem.playlist_id,
func.coalesce(func.sum(Video.length_seconds), 0),
func.count().filter(Video.live_without_length),
)
.join(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id.in_(ids))
.group_by(PlaylistItem.playlist_id)
@@ -191,11 +196,16 @@ def covers(db: Session, ids: list[int]) -> dict[int, str | None]:
)
def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None]:
"""One playlist's item count, running time and cover. An outer join, so an empty list still
answers (0, 0, None)."""
count, total = db.execute(
select(func.count(PlaylistItem.id), func.coalesce(func.sum(Video.duration_seconds), 0))
def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, int, str | None]:
"""One playlist's item count, running time, live videos the running time cannot hold
(`Video.live_without_length`) and cover. An outer join, so an empty list still answers
(0, 0, 0, None)."""
count, total, live = db.execute(
select(
func.count(PlaylistItem.id),
func.coalesce(func.sum(Video.length_seconds), 0),
func.count().filter(Video.live_without_length),
)
.select_from(PlaylistItem)
.outerjoin(Video, Video.id == PlaylistItem.video_id)
.where(PlaylistItem.playlist_id == playlist_id)
@@ -207,7 +217,7 @@ def playlist_totals(db: Session, playlist_id: int) -> tuple[int, int, str | None
.order_by(PlaylistItem.position)
.limit(1)
)
return count or 0, total or 0, cover
return count or 0, total or 0, live or 0, cover
def playlist_rows(db: Session, user: User, playlist_id: int) -> list:
+3 -1
View File
@@ -46,11 +46,12 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_
# computed here — the manager table dropped its N/S/L column.)
channel_ids = [c.id for _, c in rows]
agg: dict[str, dict] = {}
for cid, n, last_up, total_dur in queries.aggregates(db, channel_ids):
for cid, n, last_up, total_dur, live in queries.aggregates(db, channel_ids):
agg[cid] = {
"stored": n,
"last_video_at": iso(last_up),
"total_duration_seconds": int(total_dur or 0),
"live_without_length": live,
}
# Channels already in the *global* deep-backfill queue — i.e. at least one user (any
@@ -68,6 +69,7 @@ def list_channels(user: User = Depends(current_user), db: Session = Depends(get_
"stored_videos": (agg.get(ch.id) or {}).get("stored", 0),
"last_video_at": (agg.get(ch.id) or {}).get("last_video_at"),
"total_duration_seconds": (agg.get(ch.id) or {}).get("total_duration_seconds", 0),
"live_without_length": (agg.get(ch.id) or {}).get("live_without_length", 0),
"priority": sub.priority,
"hidden": sub.hidden,
"deep_requested": sub.deep_requested,
+4 -3
View File
@@ -10,6 +10,7 @@ from sqlalchemy.orm.exc import StaleDataError
from app import jobruns, quota, scheduler
from app.catalog import queries as catalog_queries, service as catalog_service
from app.catalog.models import known_length
from app.auth import current_user
from app.db import get_db, release_read_connection
from app.mailtext import lang_for_user
@@ -287,7 +288,7 @@ def get_video_detail(
"channel_title": v.channel.title if v.channel else None,
"published_at": iso(v.published_at),
"view_count": v.view_count,
"duration_seconds": v.duration_seconds,
"duration_seconds": v.length_seconds,
}
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit.
@@ -315,8 +316,8 @@ def get_video_detail(
"channel_title": snippet.get("channelTitle"),
"published_at": snippet.get("publishedAt"),
"view_count": int(views) if views is not None else None,
"duration_seconds": parse_iso8601_duration(
items[0].get("contentDetails", {}).get("duration")
"duration_seconds": known_length(
parse_iso8601_duration(items[0].get("contentDetails", {}).get("duration"))
),
}
+8 -4
View File
@@ -95,10 +95,13 @@ def _video_or_404(db: Session, video_id: str) -> None:
def _summary(
pl: "Playlist",
count: int,
total_duration: int = 0,
duration: tuple[int, int] = (0, 0),
has_video: bool | None = None,
cover: str | None = None,
) -> dict:
"""`duration` is `queries.durations`'s pair: the known running time, and the live videos it
cannot hold — the "+N live" beside it."""
total_duration, live = duration
out = {
"id": pl.id,
"name": pl.name,
@@ -108,6 +111,7 @@ def _summary(
"dirty": pl.dirty,
"item_count": count,
"total_duration_seconds": total_duration,
"live_without_length": live,
"cover_thumbnail": cover,
# When this list last CHANGED (rename, add/remove/move/reorder) — the rail's
# "recently changed first" sort reads it. See `service.touch` for why it isn't automatic.
@@ -138,7 +142,7 @@ def list_playlists(
_summary(
p,
counts.get(p.id, 0),
durations.get(p.id, 0),
durations.get(p.id, (0, 0)),
(p.id in member) if contains else None,
covers.get(p.id),
)
@@ -363,8 +367,8 @@ def rename_playlist(
raise _refused(400, "name_required")
service.rename_playlist(db, pl, name)
db.commit()
count, total, cover = queries.playlist_totals(db, pl.id)
return _summary(pl, count, total, cover=cover)
count, total, live, cover = queries.playlist_totals(db, pl.id)
return _summary(pl, count, (total, live), cover=cover)
@router.delete("/{playlist_id}")