peter
5b846136ad
YouTube reports a 24/7 stream's archive with the years it has run (prod: 122,962,914 s), and one
such video made the Watch later total read 44,073 hours. A length past a day is now no length
(`Video.length_seconds`, LENGTH_MAX_S): the API sends none, the playlist and channel totals
leave it out and count it as `live_without_length` ("+N live" beside the sum), and the duration
filter, sort and Home budget never match it. Migration 0117 moves the duration sort indexes onto
the length expression. An AST test holds who may still read the raw column.
The migration 0110 test now ends its read before the way down: 0117 builds an index
CONCURRENTLY there, which waited for that open transaction forever.
316 lines
18 KiB
Python
316 lines
18 KiB
Python
"""Catalog's tables: the shared YouTube world every user reads — channels and their videos — and
|
|
each user's footprint in it: what they subscribe to, block, explored or found by search. Only this
|
|
unit writes them; another unit reads them along a declared edge, or asks this unit's service."""
|
|
|
|
from datetime import datetime
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import (
|
|
BigInteger,
|
|
Boolean,
|
|
Computed,
|
|
DateTime,
|
|
ForeignKey,
|
|
Integer,
|
|
JSON,
|
|
String,
|
|
Text,
|
|
UniqueConstraint,
|
|
and_,
|
|
case,
|
|
func,
|
|
literal_column,
|
|
null,
|
|
)
|
|
from sqlalchemy.dialects.postgresql import TSVECTOR
|
|
from sqlalchemy.ext.hybrid import hybrid_property
|
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
|
from sqlalchemy.sql.elements import ColumnElement
|
|
|
|
from app.db import Base
|
|
from app.models import TimestampMixin
|
|
|
|
if TYPE_CHECKING:
|
|
from app.models import User
|
|
|
|
|
|
class Channel(Base, TimestampMixin):
|
|
"""A YouTube channel. Shared across all users (one channel's videos are the same
|
|
for everyone), so its expensive metadata is fetched and stored only once."""
|
|
|
|
__tablename__ = "channels"
|
|
|
|
id: Mapped[str] = mapped_column(String(32), primary_key=True) # "UC..." channel id
|
|
title: Mapped[str | None] = mapped_column(String(255))
|
|
handle: Mapped[str | None] = mapped_column(String(255))
|
|
description: Mapped[str | None] = mapped_column(Text)
|
|
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
|
uploads_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
|
subscriber_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
video_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
# Channel "About" metadata (fetched with the same channels.list call, no extra quota):
|
|
# lifetime view count, the channel's join date, banner image, and external links.
|
|
total_view_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
banner_url: Mapped[str | None] = mapped_column(String(1024))
|
|
external_links: Mapped[list | None] = mapped_column(JSON)
|
|
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
|
default_language: Mapped[str | None] = mapped_column(String(16))
|
|
country: Mapped[str | None] = mapped_column(String(8))
|
|
# Creator keyword tags (brandingSettings.channel.keywords) — a single space-separated string,
|
|
# multi-word tags quoted; the client parses it into chips for the About tab.
|
|
keywords: Mapped[str | None] = mapped_column(Text)
|
|
|
|
# Sync bookkeeping.
|
|
details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
# When the newest uploads were last read through the Data API because the RSS feed would not
|
|
# answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock.
|
|
feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
|
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# True when this channel only entered the catalog because a live YouTube search
|
|
# surfaced one of its videos (nobody subscribes to it / it has no organic upload sync).
|
|
# Lets the Library hide search-only "noise" by default.
|
|
from_search: Mapped[bool] = mapped_column(
|
|
Boolean, default=False, server_default="false", index=True
|
|
)
|
|
# True when this channel exists only because a user EXPLORED it (opened its page without
|
|
# subscribing) — an ephemeral, per-explorer addition. After a grace period the cleanup job
|
|
# hard-deletes its via_explore videos nobody keeps (no subscription, no live ExploredChannel
|
|
# row, no per-user state), and then the channel row itself once it holds no videos — unless a
|
|
# Takeout watch history shows it watched: the "watched but not followed" list names it from
|
|
# that row. Cleared when someone subscribes, here or on YouTube (the "keep" signal promotes it
|
|
# to organic).
|
|
from_explore: Mapped[bool] = mapped_column(
|
|
Boolean, default=False, server_default="false", index=True
|
|
)
|
|
|
|
videos: Mapped[list["Video"]] = relationship(back_populates="channel")
|
|
subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel")
|
|
|
|
|
|
class Subscription(Base, TimestampMixin):
|
|
"""Per-user link to a channel, with the user's own priority/visibility overrides."""
|
|
|
|
__tablename__ = "subscriptions"
|
|
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
channel_id: Mapped[str] = mapped_column(
|
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
|
)
|
|
# YouTube's subscription resource id, needed to unsubscribe via the API.
|
|
yt_subscription_id: Mapped[str | None] = mapped_column(String(128))
|
|
priority: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
hidden: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# Per-user opt-in to full-history (deep) backfill of this channel. Default off so a
|
|
# new user's unique channels don't trigger a big shared-quota burst; the deep
|
|
# scheduler only picks up channels at least one user has requested.
|
|
deep_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
subscribed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
|
|
user: Mapped["User"] = relationship(back_populates="subscriptions")
|
|
channel: Mapped["Channel"] = relationship(back_populates="subscriptions")
|
|
|
|
|
|
# Video.live_status values that mean "currently a live/upcoming broadcast" — transient states
|
|
# hidden from feeds/search and revisited by the refresh job until they settle into a VOD.
|
|
LIVE_OR_UPCOMING = ("live", "upcoming")
|
|
|
|
# A YouTube length past a day is not a length. A 24/7 stream's archive (`was_live`) reports the years
|
|
# the stream has run (measured on prod, 2026-11-06: 122,962,914 s), and one such video made a Watch
|
|
# later total read 44,073 hours. YouTube caps an upload at 12 hours, so nothing real is lost. Such a
|
|
# length is unknown wherever a length is used: the API sends none (`Video.length_seconds`), sums leave
|
|
# it out and count it as `live_without_length`, and the duration filter, sort and Home budget never
|
|
# match it. The raw `duration_seconds` stays for what it does prove — that the video is not a Short —
|
|
# and for the jobs that ask whether YouTube has answered yet (test_video_length holds who reads it).
|
|
LENGTH_MAX_S = 24 * 3600
|
|
|
|
|
|
def known_length(seconds: int | None) -> int | None:
|
|
"""`seconds` as a video's length, or None when it is not one (see LENGTH_MAX_S)."""
|
|
return seconds if seconds is not None and seconds <= LENGTH_MAX_S else None
|
|
|
|
|
|
class Video(Base, TimestampMixin):
|
|
"""A YouTube video. Shared across users; per-user state lives elsewhere."""
|
|
|
|
__tablename__ = "videos"
|
|
|
|
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
|
# No standalone channel_id index: the composite `ix_videos_channel_pub_cover`
|
|
# (channel_id, published_at DESC NULLS LAST, id) from migration 0095 leads with channel_id, so it
|
|
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) and the per-channel feed
|
|
# sort. Its INCLUDEd columns make the channel manager's sums and the feed sidebar's counts Index Only
|
|
# Scans (no heap): a column those queries start reading belongs in it (test_channel_list_index_only).
|
|
# Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
|
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
|
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
|
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
|
description: Mapped[str | None] = mapped_column(Text)
|
|
# NOT NULL (migration 0068): in reality every video carries a publish date, and a nullable key
|
|
# forced the feed's keyset predicate into a non-sargable `... OR published_at IS NULL` disjunction
|
|
# that blocked the ix_videos_published_at_desc range (deep pages scanned from the top and filtered —
|
|
# PERF-15). server_default now() is the safety net for the rare date-less ingest (the enrich fills
|
|
# the real date); the ix_videos_published_at index stays for the `oldest` sort.
|
|
published_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), index=True, nullable=False, server_default=func.now()
|
|
)
|
|
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
|
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
|
|
# ORDER BY, one per direction (and per ICU collation for title): migration 0096 (the duration
|
|
# pair since 0117, on `length_seconds`), held to `feedquery.order_by` by test_feed_sort_indexes.
|
|
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
|
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
category_id: Mapped[int | None] = mapped_column(Integer)
|
|
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
|
default_language: Mapped[str | None] = mapped_column(String(16))
|
|
detected_language: Mapped[str | None] = mapped_column(String(16))
|
|
# Creator-supplied keyword tags (snippet.tags joined by spaces) — fetched for free with the
|
|
# snippet we already request. Indexed into search_vector so the feed search matches the
|
|
# uploader's own keywords, not just words in the title.
|
|
keywords: Mapped[str | None] = mapped_column(Text)
|
|
# Accumulated distinct live-search queries that surfaced this video (across all users) —
|
|
# folds YouTube's own relevance judgement into the local search: a video YouTube returned
|
|
# for "ti amo magyarul" becomes findable by that query locally even if its title lacks it.
|
|
search_terms: Mapped[str | None] = mapped_column(Text)
|
|
# No btree here or on the flag columns below: migration 0069 (PERF-3) dropped the never-picked
|
|
# boolean btrees and replaced the useful ones with partial indexes over the target subset
|
|
# (via_search=true, NOT shorts_probed, unavailable_since IS NOT NULL) — migration-defined, like the
|
|
# search_vector GIN.
|
|
is_short: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# Whether the youtube.com/shorts/<id> probe has run for this video.
|
|
shorts_probed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# True when this video first entered the catalog via a live YouTube search (not an
|
|
# organic channel/playlist sync). Used to hide search-discovered "noise" from the
|
|
# Library by default; left False by the organic ingest paths.
|
|
via_search: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# True when this video entered the catalog because a user EXPLORED its (un-subscribed)
|
|
# channel. Like via_search it's hidden from the shared Library by default, but it's
|
|
# additionally PER-USER private: a via_explore video is only visible to users who
|
|
# explored or subscribe to its channel (a gate in the feed query), so one user's
|
|
# curiosity never floods everyone else's catalog. Left False by the organic ingest paths.
|
|
via_explore: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# none | live | upcoming | premiere | was_live
|
|
live_status: Mapped[str] = mapped_column(
|
|
String(16), default="none", server_default="none", index=True
|
|
)
|
|
enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
# videos.list?part=status fields (populated during enrichment); used by the
|
|
# maintenance/validation job to decide whether a video is playable anywhere.
|
|
embeddable: Mapped[bool | None] = mapped_column(Boolean)
|
|
privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private
|
|
upload_status: Mapped[str | None] = mapped_column(
|
|
String(16)
|
|
) # processed|failed|rejected|deleted
|
|
# Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation
|
|
# (least-recently-checked first); unavailable_since marks a video as currently
|
|
# unplayable (hidden from the feed immediately, hard-deleted after a grace period);
|
|
# unavailable_reason records why (deleted|private|paywalled|abandoned|stuck_live).
|
|
# last_checked_at: no plain btree — migration 0069 (PERF-18) replaced it with the partial
|
|
# ix_videos_recheck_order (last_checked_at NULLS FIRST WHERE unavailable_since IS NULL) that the
|
|
# revalidate job's exact ORDER BY rides; unavailable_since is a partial WHERE IS NOT NULL (PERF-3).
|
|
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
|
|
|
|
@hybrid_property
|
|
def length_seconds(self) -> int | None:
|
|
"""The video's length, or None when it is unknown — not fetched yet, or not a length at all
|
|
(`LENGTH_MAX_S`). What every reader of a length reads."""
|
|
return known_length(self.duration_seconds)
|
|
|
|
@length_seconds.inplace.expression
|
|
@classmethod
|
|
def _length_seconds_expression(cls) -> ColumnElement[int | None]:
|
|
# The limit is a literal, not a bound parameter: the duration sort indexes (migration 0117)
|
|
# are built on this very expression, and Postgres matches an index to a query's constant only.
|
|
return case(
|
|
(cls.duration_seconds > literal_column(str(LENGTH_MAX_S)), null()),
|
|
else_=cls.duration_seconds,
|
|
)
|
|
|
|
@hybrid_property
|
|
def live_without_length(self) -> bool:
|
|
"""A live video (on air, upcoming, a premiere or an archive) whose length is unknown: what a
|
|
total leaves out and names as "+N live"."""
|
|
return self.live_status != "none" and self.length_seconds is None
|
|
|
|
@live_without_length.inplace.expression
|
|
@classmethod
|
|
def _live_without_length_expression(cls) -> ColumnElement[bool]:
|
|
return and_(cls.live_status != "none", cls.length_seconds.is_(None))
|
|
|
|
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
|
|
# creator keywords + search queries (B) > truncated description (C). ts_rank honours the
|
|
# weights so title matches outrank description matches. Backed by a GIN index (migration
|
|
# 0032). The accent-insensitive `unaccent_simple` config comes from migration 0031.
|
|
search_vector: Mapped[object | None] = mapped_column(
|
|
TSVECTOR,
|
|
Computed(
|
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(title, '')), 'A') || "
|
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(keywords, '') || ' ' || "
|
|
"coalesce(search_terms, '')), 'B') || "
|
|
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(description, ''), 1000)), 'C')",
|
|
persisted=True,
|
|
),
|
|
)
|
|
|
|
channel: Mapped["Channel"] = relationship(back_populates="videos")
|
|
|
|
|
|
class SearchFind(Base, TimestampMixin):
|
|
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
|
|
|
Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source
|
|
filter — distinct from the GLOBAL `Video.via_search` provenance, which marks a video as
|
|
search-discovered by *anyone* and drives the shared Library's Source filter."""
|
|
|
|
__tablename__ = "search_finds"
|
|
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
|
|
|
|
|
class ExploredChannel(Base, TimestampMixin):
|
|
"""Per-user record of a channel the user opened the page of WITHOUT subscribing.
|
|
|
|
Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the
|
|
user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the
|
|
explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period
|
|
purge: of a from_explore channel with no live ExploredChannel row (and no subscription) the
|
|
cleanup job hard-deletes the videos without per-user state, then the channel once it holds none
|
|
and no Takeout watch history shows it watched. Subscribing is the "keep"
|
|
signal — it promotes the channel to organic and these rows become irrelevant. `created_at`
|
|
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
|
|
|
|
__tablename__ = "explored_channels"
|
|
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
channel_id: Mapped[str] = mapped_column(
|
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
|
)
|
|
|
|
|
|
class BlockedChannel(Base, TimestampMixin):
|
|
"""Per-user channel blocklist. Videos from a blocked channel are excluded from this user's
|
|
live YouTube search (and not ingested on their behalf) and hidden from their feed / explore
|
|
views — a personal "never show me this channel again" that doesn't affect other users."""
|
|
|
|
__tablename__ = "blocked_channels"
|
|
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
channel_id: Mapped[str] = mapped_column(
|
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
|
)
|