"""Catalog's tables: the shared YouTube world every user reads — channels and their videos — and each user's footprint in it: what they subscribe to, block, explored or found by search. Only this unit writes them; another unit reads them along a declared edge, or asks this unit's service.""" from datetime import datetime from typing import TYPE_CHECKING from sqlalchemy import ( BigInteger, Boolean, Computed, DateTime, ForeignKey, Integer, JSON, String, Text, UniqueConstraint, func, ) from sqlalchemy.dialects.postgresql import TSVECTOR from sqlalchemy.orm import Mapped, mapped_column, relationship from app.db import Base from app.models import TimestampMixin if TYPE_CHECKING: from app.models import User class Channel(Base, TimestampMixin): """A YouTube channel. Shared across all users (one channel's videos are the same for everyone), so its expensive metadata is fetched and stored only once.""" __tablename__ = "channels" id: Mapped[str] = mapped_column(String(32), primary_key=True) # "UC..." channel id title: Mapped[str | None] = mapped_column(String(255)) handle: Mapped[str | None] = mapped_column(String(255)) description: Mapped[str | None] = mapped_column(Text) thumbnail_url: Mapped[str | None] = mapped_column(String(1024)) uploads_playlist_id: Mapped[str | None] = mapped_column(String(64)) subscriber_count: Mapped[int | None] = mapped_column(BigInteger) video_count: Mapped[int | None] = mapped_column(BigInteger) # Channel "About" metadata (fetched with the same channels.list call, no extra quota): # lifetime view count, the channel's join date, banner image, and external links. total_view_count: Mapped[int | None] = mapped_column(BigInteger) published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) banner_url: Mapped[str | None] = mapped_column(String(1024)) external_links: Mapped[list | None] = mapped_column(JSON) topic_categories: Mapped[list | None] = mapped_column(JSON) default_language: Mapped[str | None] = mapped_column(String(16)) country: Mapped[str | None] = mapped_column(String(8)) # Creator keyword tags (brandingSettings.channel.keywords) — a single space-separated string, # multi-word tags quoted; the client parses it into chips for the About tab. keywords: Mapped[str | None] = mapped_column(Text) # Sync bookkeeping. details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) # When the newest uploads were last read through the Data API because the RSS feed would not # answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock. feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) backfill_cursor: Mapped[str | None] = mapped_column(String(128)) backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # True when this channel only entered the catalog because a live YouTube search # surfaced one of its videos (nobody subscribes to it / it has no organic upload sync). # Lets the Library hide search-only "noise" by default. from_search: Mapped[bool] = mapped_column( Boolean, default=False, server_default="false", index=True ) # True when this channel exists only because a user EXPLORED it (opened its page without # subscribing) — an ephemeral, per-explorer addition. The cleanup job hard-deletes such a # channel (and its via_explore videos) once nobody keeps it (no subscription, no live # ExploredChannel row, no per-user state) after a grace period. Cleared when someone # subscribes (the "keep" signal promotes it to organic). from_explore: Mapped[bool] = mapped_column( Boolean, default=False, server_default="false", index=True ) videos: Mapped[list["Video"]] = relationship(back_populates="channel") subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel") class Subscription(Base, TimestampMixin): """Per-user link to a channel, with the user's own priority/visibility overrides.""" __tablename__ = "subscriptions" __table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),) id: Mapped[int] = mapped_column(primary_key=True) user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) channel_id: Mapped[str] = mapped_column( ForeignKey("channels.id", ondelete="CASCADE"), index=True ) # YouTube's subscription resource id, needed to unsubscribe via the API. yt_subscription_id: Mapped[str | None] = mapped_column(String(128)) priority: Mapped[int] = mapped_column(Integer, default=0, server_default="0") hidden: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # Per-user opt-in to full-history (deep) backfill of this channel. Default off so a # new user's unique channels don't trigger a big shared-quota burst; the deep # scheduler only picks up channels at least one user has requested. deep_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") subscribed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) user: Mapped["User"] = relationship(back_populates="subscriptions") channel: Mapped["Channel"] = relationship(back_populates="subscriptions") # Video.live_status values that mean "currently a live/upcoming broadcast" — transient states # hidden from feeds/search and revisited by the refresh job until they settle into a VOD. LIVE_OR_UPCOMING = ("live", "upcoming") class Video(Base, TimestampMixin): """A YouTube video. Shared across users; per-user state lives elsewhere.""" __tablename__ = "videos" id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id # No standalone channel_id index: the composite `ix_videos_channel_pub_cover` # (channel_id, published_at DESC NULLS LAST, id) from migration 0095 leads with channel_id, so it # serves every channel_id lookup (FK cascade, sync upsert, channel-detail) and the per-channel feed # sort. Its INCLUDEd columns make the channel manager's sums and the feed sidebar's counts Index Only # Scans (no heap): a column those queries start reading belongs in it (test_channel_list_index_only). # Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`. channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE")) title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles) original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved description: Mapped[str | None] = mapped_column(Text) # NOT NULL (migration 0068): in reality every video carries a publish date, and a nullable key # forced the feed's keyset predicate into a non-sargable `... OR published_at IS NULL` disjunction # that blocked the ix_videos_published_at_desc range (deep pages scanned from the top and filtered — # PERF-15). server_default now() is the safety net for the rare date-less ingest (the enrich fills # the real date); the ix_videos_published_at index stays for the `oldest` sort. published_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), index=True, nullable=False, server_default=func.now() ) thumbnail_url: Mapped[str | None] = mapped_column(String(1024)) # The feed's views / duration / title sorts each ride a migration-defined index in their exact # ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to # `feedquery.order_by` by test_feed_sort_indexes. duration_seconds: Mapped[int | None] = mapped_column(Integer) view_count: Mapped[int | None] = mapped_column(BigInteger) like_count: Mapped[int | None] = mapped_column(BigInteger) category_id: Mapped[int | None] = mapped_column(Integer) topic_categories: Mapped[list | None] = mapped_column(JSON) default_language: Mapped[str | None] = mapped_column(String(16)) detected_language: Mapped[str | None] = mapped_column(String(16)) # Creator-supplied keyword tags (snippet.tags joined by spaces) — fetched for free with the # snippet we already request. Indexed into search_vector so the feed search matches the # uploader's own keywords, not just words in the title. keywords: Mapped[str | None] = mapped_column(Text) # Accumulated distinct live-search queries that surfaced this video (across all users) — # folds YouTube's own relevance judgement into the local search: a video YouTube returned # for "ti amo magyarul" becomes findable by that query locally even if its title lacks it. search_terms: Mapped[str | None] = mapped_column(Text) # No btree here or on the flag columns below: migration 0069 (PERF-3) dropped the never-picked # boolean btrees and replaced the useful ones with partial indexes over the target subset # (via_search=true, NOT shorts_probed, unavailable_since IS NOT NULL) — migration-defined, like the # search_vector GIN. is_short: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # Whether the youtube.com/shorts/ probe has run for this video. shorts_probed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # True when this video first entered the catalog via a live YouTube search (not an # organic channel/playlist sync). Used to hide search-discovered "noise" from the # Library by default; left False by the organic ingest paths. via_search: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # True when this video entered the catalog because a user EXPLORED its (un-subscribed) # channel. Like via_search it's hidden from the shared Library by default, but it's # additionally PER-USER private: a via_explore video is only visible to users who # explored or subscribe to its channel (a gate in the feed query), so one user's # curiosity never floods everyone else's catalog. Left False by the organic ingest paths. via_explore: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false") # none | live | upcoming | premiere | was_live live_status: Mapped[str] = mapped_column( String(16), default="none", server_default="none", index=True ) enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) # videos.list?part=status fields (populated during enrichment); used by the # maintenance/validation job to decide whether a video is playable anywhere. embeddable: Mapped[bool | None] = mapped_column(Boolean) privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private upload_status: Mapped[str | None] = mapped_column( String(16) ) # processed|failed|rejected|deleted # Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation # (least-recently-checked first); unavailable_since marks a video as currently # unplayable (hidden from the feed immediately, hard-deleted after a grace period); # unavailable_reason records why (deleted|private|paywalled|abandoned|stuck_live). # last_checked_at: no plain btree — migration 0069 (PERF-18) replaced it with the partial # ix_videos_recheck_order (last_checked_at NULLS FIRST WHERE unavailable_since IS NULL) that the # revalidate job's exact ORDER BY rides; unavailable_since is a partial WHERE IS NOT NULL (PERF-3). last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) unavailable_reason: Mapped[str | None] = mapped_column(String(24)) # Weighted full-text search document (DB-generated, never written by the app): title (A) > # creator keywords + search queries (B) > truncated description (C). ts_rank honours the # weights so title matches outrank description matches. Backed by a GIN index (migration # 0032). The accent-insensitive `unaccent_simple` config comes from migration 0031. search_vector: Mapped[object | None] = mapped_column( TSVECTOR, Computed( "setweight(to_tsvector('public.unaccent_simple', coalesce(title, '')), 'A') || " "setweight(to_tsvector('public.unaccent_simple', coalesce(keywords, '') || ' ' || " "coalesce(search_terms, '')), 'B') || " "setweight(to_tsvector('public.unaccent_simple', left(coalesce(description, ''), 1000)), 'C')", persisted=True, ), ) channel: Mapped["Channel"] = relationship(back_populates="videos") class SearchFind(Base, TimestampMixin): """Per-user record of a video the user surfaced via their own live YouTube search. Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source filter — distinct from the GLOBAL `Video.via_search` provenance, which marks a video as search-discovered by *anyone* and drives the shared Library's Source filter.""" __tablename__ = "search_finds" __table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),) id: Mapped[int] = mapped_column(primary_key=True) user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True) class ExploredChannel(Base, TimestampMixin): """Per-user record of a channel the user opened the page of WITHOUT subscribing. Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period purge: a from_explore channel with no live ExploredChannel row (and no subscription / no per-user state on its videos) is hard-deleted by the cleanup job. Subscribing is the "keep" signal — it promotes the channel to organic and these rows become irrelevant. `created_at` (from TimestampMixin) is the grace clock; a re-visit refreshes it.""" __tablename__ = "explored_channels" __table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),) id: Mapped[int] = mapped_column(primary_key=True) user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) channel_id: Mapped[str] = mapped_column( ForeignKey("channels.id", ondelete="CASCADE"), index=True ) class BlockedChannel(Base, TimestampMixin): """Per-user channel blocklist. Videos from a blocked channel are excluded from this user's live YouTube search (and not ingested on their behalf) and hidden from their feed / explore views — a personal "never show me this channel again" that doesn't affect other users.""" __tablename__ = "blocked_channels" __table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),) id: Mapped[int] = mapped_column(primary_key=True) user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE")) channel_id: Mapped[str] = mapped_column( ForeignKey("channels.id", ondelete="CASCADE"), index=True )