fix(catalog): a 24/7 stream archive's years-long length is unknown, not summed or sorted by
YouTube reports a 24/7 stream's archive with the years it has run (prod: 122,962,914 s), and one
such video made the Watch later total read 44,073 hours. A length past a day is now no length
(`Video.length_seconds`, LENGTH_MAX_S): the API sends none, the playlist and channel totals
leave it out and count it as `live_without_length` ("+N live" beside the sum), and the duration
filter, sort and Home budget never match it. Migration 0117 moves the duration sort indexes onto
the length expression. An AST test holds who may still read the raw column.
The migration 0110 test now ends its read before the way down: 0117 builds an index
CONCURRENTLY there, which waited for that open transaction forever.
This commit is contained in:
@@ -16,10 +16,16 @@ from sqlalchemy import (
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
and_,
|
||||
case,
|
||||
func,
|
||||
literal_column,
|
||||
null,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import TSVECTOR
|
||||
from sqlalchemy.ext.hybrid import hybrid_property
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from sqlalchemy.sql.elements import ColumnElement
|
||||
|
||||
from app.db import Base
|
||||
from app.models import TimestampMixin
|
||||
@@ -114,6 +120,20 @@ class Subscription(Base, TimestampMixin):
|
||||
# hidden from feeds/search and revisited by the refresh job until they settle into a VOD.
|
||||
LIVE_OR_UPCOMING = ("live", "upcoming")
|
||||
|
||||
# A YouTube length past a day is not a length. A 24/7 stream's archive (`was_live`) reports the years
|
||||
# the stream has run (measured on prod, 2026-11-06: 122,962,914 s), and one such video made a Watch
|
||||
# later total read 44,073 hours. YouTube caps an upload at 12 hours, so nothing real is lost. Such a
|
||||
# length is unknown wherever a length is used: the API sends none (`Video.length_seconds`), sums leave
|
||||
# it out and count it as `live_without_length`, and the duration filter, sort and Home budget never
|
||||
# match it. The raw `duration_seconds` stays for what it does prove — that the video is not a Short —
|
||||
# and for the jobs that ask whether YouTube has answered yet (test_video_length holds who reads it).
|
||||
LENGTH_MAX_S = 24 * 3600
|
||||
|
||||
|
||||
def known_length(seconds: int | None) -> int | None:
|
||||
"""`seconds` as a video's length, or None when it is not one (see LENGTH_MAX_S)."""
|
||||
return seconds if seconds is not None and seconds <= LENGTH_MAX_S else None
|
||||
|
||||
|
||||
class Video(Base, TimestampMixin):
|
||||
"""A YouTube video. Shared across users; per-user state lives elsewhere."""
|
||||
@@ -141,8 +161,8 @@ class Video(Base, TimestampMixin):
|
||||
)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
|
||||
# ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to
|
||||
# `feedquery.order_by` by test_feed_sort_indexes.
|
||||
# ORDER BY, one per direction (and per ICU collation for title): migration 0096 (the duration
|
||||
# pair since 0117, on `length_seconds`), held to `feedquery.order_by` by test_feed_sort_indexes.
|
||||
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
@@ -198,6 +218,33 @@ class Video(Base, TimestampMixin):
|
||||
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
|
||||
|
||||
@hybrid_property
|
||||
def length_seconds(self) -> int | None:
|
||||
"""The video's length, or None when it is unknown — not fetched yet, or not a length at all
|
||||
(`LENGTH_MAX_S`). What every reader of a length reads."""
|
||||
return known_length(self.duration_seconds)
|
||||
|
||||
@length_seconds.inplace.expression
|
||||
@classmethod
|
||||
def _length_seconds_expression(cls) -> ColumnElement[int | None]:
|
||||
# The limit is a literal, not a bound parameter: the duration sort indexes (migration 0117)
|
||||
# are built on this very expression, and Postgres matches an index to a query's constant only.
|
||||
return case(
|
||||
(cls.duration_seconds > literal_column(str(LENGTH_MAX_S)), null()),
|
||||
else_=cls.duration_seconds,
|
||||
)
|
||||
|
||||
@hybrid_property
|
||||
def live_without_length(self) -> bool:
|
||||
"""A live video (on air, upcoming, a premiere or an archive) whose length is unknown: what a
|
||||
total leaves out and names as "+N live"."""
|
||||
return self.live_status != "none" and self.length_seconds is None
|
||||
|
||||
@live_without_length.inplace.expression
|
||||
@classmethod
|
||||
def _live_without_length_expression(cls) -> ColumnElement[bool]:
|
||||
return and_(cls.live_status != "none", cls.length_seconds.is_(None))
|
||||
|
||||
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
|
||||
# creator keywords + search queries (B) > truncated description (C). ts_rank honours the
|
||||
# weights so title matches outrank description matches. Backed by a GIN index (migration
|
||||
|
||||
Reference in New Issue
Block a user