peter
bc8fe2cc6c
A subscription pulled in from YouTube left an explored channel flagged as an exploration's, so its later videos came in hidden from the Library with their watched marks waiting for good; a subscribe landing while an exploration's videos came in left those flagged too. One step, settle_explored, now holds the rule by who follows the channel now: the subscribe here, the subscription import, the watched-channel videos and the explore page all end with it. Migration 0116 clears what the gap left behind. The list's "may still grow" count no longer counts a channel it already shows from a stub row, and the comments on the cleanup's keep rule and on the in-memory daily shares say what they do.
269 lines
16 KiB
Python
269 lines
16 KiB
Python
"""Catalog's tables: the shared YouTube world every user reads — channels and their videos — and
|
|
each user's footprint in it: what they subscribe to, block, explored or found by search. Only this
|
|
unit writes them; another unit reads them along a declared edge, or asks this unit's service."""
|
|
|
|
from datetime import datetime
|
|
from typing import TYPE_CHECKING
|
|
|
|
from sqlalchemy import (
|
|
BigInteger,
|
|
Boolean,
|
|
Computed,
|
|
DateTime,
|
|
ForeignKey,
|
|
Integer,
|
|
JSON,
|
|
String,
|
|
Text,
|
|
UniqueConstraint,
|
|
func,
|
|
)
|
|
from sqlalchemy.dialects.postgresql import TSVECTOR
|
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
|
|
|
from app.db import Base
|
|
from app.models import TimestampMixin
|
|
|
|
if TYPE_CHECKING:
|
|
from app.models import User
|
|
|
|
|
|
class Channel(Base, TimestampMixin):
|
|
"""A YouTube channel. Shared across all users (one channel's videos are the same
|
|
for everyone), so its expensive metadata is fetched and stored only once."""
|
|
|
|
__tablename__ = "channels"
|
|
|
|
id: Mapped[str] = mapped_column(String(32), primary_key=True) # "UC..." channel id
|
|
title: Mapped[str | None] = mapped_column(String(255))
|
|
handle: Mapped[str | None] = mapped_column(String(255))
|
|
description: Mapped[str | None] = mapped_column(Text)
|
|
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
|
uploads_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
|
subscriber_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
video_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
# Channel "About" metadata (fetched with the same channels.list call, no extra quota):
|
|
# lifetime view count, the channel's join date, banner image, and external links.
|
|
total_view_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
banner_url: Mapped[str | None] = mapped_column(String(1024))
|
|
external_links: Mapped[list | None] = mapped_column(JSON)
|
|
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
|
default_language: Mapped[str | None] = mapped_column(String(16))
|
|
country: Mapped[str | None] = mapped_column(String(8))
|
|
# Creator keyword tags (brandingSettings.channel.keywords) — a single space-separated string,
|
|
# multi-word tags quoted; the client parses it into chips for the About tab.
|
|
keywords: Mapped[str | None] = mapped_column(Text)
|
|
|
|
# Sync bookkeeping.
|
|
details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
# When the newest uploads were last read through the Data API because the RSS feed would not
|
|
# answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock.
|
|
feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
|
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# True when this channel only entered the catalog because a live YouTube search
|
|
# surfaced one of its videos (nobody subscribes to it / it has no organic upload sync).
|
|
# Lets the Library hide search-only "noise" by default.
|
|
from_search: Mapped[bool] = mapped_column(
|
|
Boolean, default=False, server_default="false", index=True
|
|
)
|
|
# True when this channel exists only because a user EXPLORED it (opened its page without
|
|
# subscribing) — an ephemeral, per-explorer addition. After a grace period the cleanup job
|
|
# hard-deletes its via_explore videos nobody keeps (no subscription, no live ExploredChannel
|
|
# row, no per-user state), and then the channel row itself once it holds no videos — unless a
|
|
# Takeout watch history shows it watched: the "watched but not followed" list names it from
|
|
# that row. Cleared when someone subscribes, here or on YouTube (the "keep" signal promotes it
|
|
# to organic).
|
|
from_explore: Mapped[bool] = mapped_column(
|
|
Boolean, default=False, server_default="false", index=True
|
|
)
|
|
|
|
videos: Mapped[list["Video"]] = relationship(back_populates="channel")
|
|
subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel")
|
|
|
|
|
|
class Subscription(Base, TimestampMixin):
|
|
"""Per-user link to a channel, with the user's own priority/visibility overrides."""
|
|
|
|
__tablename__ = "subscriptions"
|
|
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
channel_id: Mapped[str] = mapped_column(
|
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
|
)
|
|
# YouTube's subscription resource id, needed to unsubscribe via the API.
|
|
yt_subscription_id: Mapped[str | None] = mapped_column(String(128))
|
|
priority: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
hidden: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# Per-user opt-in to full-history (deep) backfill of this channel. Default off so a
|
|
# new user's unique channels don't trigger a big shared-quota burst; the deep
|
|
# scheduler only picks up channels at least one user has requested.
|
|
deep_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
subscribed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
|
|
user: Mapped["User"] = relationship(back_populates="subscriptions")
|
|
channel: Mapped["Channel"] = relationship(back_populates="subscriptions")
|
|
|
|
|
|
# Video.live_status values that mean "currently a live/upcoming broadcast" — transient states
|
|
# hidden from feeds/search and revisited by the refresh job until they settle into a VOD.
|
|
LIVE_OR_UPCOMING = ("live", "upcoming")
|
|
|
|
|
|
class Video(Base, TimestampMixin):
|
|
"""A YouTube video. Shared across users; per-user state lives elsewhere."""
|
|
|
|
__tablename__ = "videos"
|
|
|
|
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
|
# No standalone channel_id index: the composite `ix_videos_channel_pub_cover`
|
|
# (channel_id, published_at DESC NULLS LAST, id) from migration 0095 leads with channel_id, so it
|
|
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) and the per-channel feed
|
|
# sort. Its INCLUDEd columns make the channel manager's sums and the feed sidebar's counts Index Only
|
|
# Scans (no heap): a column those queries start reading belongs in it (test_channel_list_index_only).
|
|
# Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
|
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
|
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
|
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
|
description: Mapped[str | None] = mapped_column(Text)
|
|
# NOT NULL (migration 0068): in reality every video carries a publish date, and a nullable key
|
|
# forced the feed's keyset predicate into a non-sargable `... OR published_at IS NULL` disjunction
|
|
# that blocked the ix_videos_published_at_desc range (deep pages scanned from the top and filtered —
|
|
# PERF-15). server_default now() is the safety net for the rare date-less ingest (the enrich fills
|
|
# the real date); the ix_videos_published_at index stays for the `oldest` sort.
|
|
published_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), index=True, nullable=False, server_default=func.now()
|
|
)
|
|
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
|
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
|
|
# ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to
|
|
# `feedquery.order_by` by test_feed_sort_indexes.
|
|
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
|
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
|
category_id: Mapped[int | None] = mapped_column(Integer)
|
|
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
|
default_language: Mapped[str | None] = mapped_column(String(16))
|
|
detected_language: Mapped[str | None] = mapped_column(String(16))
|
|
# Creator-supplied keyword tags (snippet.tags joined by spaces) — fetched for free with the
|
|
# snippet we already request. Indexed into search_vector so the feed search matches the
|
|
# uploader's own keywords, not just words in the title.
|
|
keywords: Mapped[str | None] = mapped_column(Text)
|
|
# Accumulated distinct live-search queries that surfaced this video (across all users) —
|
|
# folds YouTube's own relevance judgement into the local search: a video YouTube returned
|
|
# for "ti amo magyarul" becomes findable by that query locally even if its title lacks it.
|
|
search_terms: Mapped[str | None] = mapped_column(Text)
|
|
# No btree here or on the flag columns below: migration 0069 (PERF-3) dropped the never-picked
|
|
# boolean btrees and replaced the useful ones with partial indexes over the target subset
|
|
# (via_search=true, NOT shorts_probed, unavailable_since IS NOT NULL) — migration-defined, like the
|
|
# search_vector GIN.
|
|
is_short: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# Whether the youtube.com/shorts/<id> probe has run for this video.
|
|
shorts_probed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# True when this video first entered the catalog via a live YouTube search (not an
|
|
# organic channel/playlist sync). Used to hide search-discovered "noise" from the
|
|
# Library by default; left False by the organic ingest paths.
|
|
via_search: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# True when this video entered the catalog because a user EXPLORED its (un-subscribed)
|
|
# channel. Like via_search it's hidden from the shared Library by default, but it's
|
|
# additionally PER-USER private: a via_explore video is only visible to users who
|
|
# explored or subscribe to its channel (a gate in the feed query), so one user's
|
|
# curiosity never floods everyone else's catalog. Left False by the organic ingest paths.
|
|
via_explore: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# none | live | upcoming | premiere | was_live
|
|
live_status: Mapped[str] = mapped_column(
|
|
String(16), default="none", server_default="none", index=True
|
|
)
|
|
enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
# videos.list?part=status fields (populated during enrichment); used by the
|
|
# maintenance/validation job to decide whether a video is playable anywhere.
|
|
embeddable: Mapped[bool | None] = mapped_column(Boolean)
|
|
privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private
|
|
upload_status: Mapped[str | None] = mapped_column(
|
|
String(16)
|
|
) # processed|failed|rejected|deleted
|
|
# Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation
|
|
# (least-recently-checked first); unavailable_since marks a video as currently
|
|
# unplayable (hidden from the feed immediately, hard-deleted after a grace period);
|
|
# unavailable_reason records why (deleted|private|paywalled|abandoned|stuck_live).
|
|
# last_checked_at: no plain btree — migration 0069 (PERF-18) replaced it with the partial
|
|
# ix_videos_recheck_order (last_checked_at NULLS FIRST WHERE unavailable_since IS NULL) that the
|
|
# revalidate job's exact ORDER BY rides; unavailable_since is a partial WHERE IS NOT NULL (PERF-3).
|
|
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
|
|
|
|
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
|
|
# creator keywords + search queries (B) > truncated description (C). ts_rank honours the
|
|
# weights so title matches outrank description matches. Backed by a GIN index (migration
|
|
# 0032). The accent-insensitive `unaccent_simple` config comes from migration 0031.
|
|
search_vector: Mapped[object | None] = mapped_column(
|
|
TSVECTOR,
|
|
Computed(
|
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(title, '')), 'A') || "
|
|
"setweight(to_tsvector('public.unaccent_simple', coalesce(keywords, '') || ' ' || "
|
|
"coalesce(search_terms, '')), 'B') || "
|
|
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(description, ''), 1000)), 'C')",
|
|
persisted=True,
|
|
),
|
|
)
|
|
|
|
channel: Mapped["Channel"] = relationship(back_populates="videos")
|
|
|
|
|
|
class SearchFind(Base, TimestampMixin):
|
|
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
|
|
|
Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source
|
|
filter — distinct from the GLOBAL `Video.via_search` provenance, which marks a video as
|
|
search-discovered by *anyone* and drives the shared Library's Source filter."""
|
|
|
|
__tablename__ = "search_finds"
|
|
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
|
|
|
|
|
class ExploredChannel(Base, TimestampMixin):
|
|
"""Per-user record of a channel the user opened the page of WITHOUT subscribing.
|
|
|
|
Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the
|
|
user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the
|
|
explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period
|
|
purge: of a from_explore channel with no live ExploredChannel row (and no subscription) the
|
|
cleanup job hard-deletes the videos without per-user state, then the channel once it holds none
|
|
and no Takeout watch history shows it watched. Subscribing is the "keep"
|
|
signal — it promotes the channel to organic and these rows become irrelevant. `created_at`
|
|
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
|
|
|
|
__tablename__ = "explored_channels"
|
|
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
channel_id: Mapped[str] = mapped_column(
|
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
|
)
|
|
|
|
|
|
class BlockedChannel(Base, TimestampMixin):
|
|
"""Per-user channel blocklist. Videos from a blocked channel are excluded from this user's
|
|
live YouTube search (and not ingested on their behalf) and hidden from their feed / explore
|
|
views — a personal "never show me this channel again" that doesn't affect other users."""
|
|
|
|
__tablename__ = "blocked_channels"
|
|
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
channel_id: Mapped[str] = mapped_column(
|
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
|
)
|