refactor(catalog): the unit's tables move into catalog/models
Channel, Subscription, Video, SearchFind, ExploredChannel and BlockedChannel (and LIVE_OR_UPCOMING) leave app/models.py for app/catalog/models.py; app.tables registers them. The User↔Subscription relationship names its other side as a type only. No schema change.
This commit is contained in:
+4
-237
@@ -1,4 +1,5 @@
|
||||
from datetime import date, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
@@ -26,6 +27,9 @@ from app.db import Base
|
||||
from app.runstatus import RunStatus, RunTrigger, sql_in
|
||||
from app.utils import canonical_email
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.catalog.models import Subscription
|
||||
|
||||
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
||||
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
||||
# `User.display_name` (their Google profile name), `DownloadJob.display_name` and
|
||||
@@ -220,243 +224,6 @@ class Invite(Base, CanonicalEmailMixin):
|
||||
lang: Mapped[str | None] = mapped_column(String(8))
|
||||
|
||||
|
||||
class Channel(Base, TimestampMixin):
|
||||
"""A YouTube channel. Shared across all users (one channel's videos are the same
|
||||
for everyone), so its expensive metadata is fetched and stored only once."""
|
||||
|
||||
__tablename__ = "channels"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True) # "UC..." channel id
|
||||
title: Mapped[str | None] = mapped_column(String(255))
|
||||
handle: Mapped[str | None] = mapped_column(String(255))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
uploads_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
||||
subscriber_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
video_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
# Channel "About" metadata (fetched with the same channels.list call, no extra quota):
|
||||
# lifetime view count, the channel's join date, banner image, and external links.
|
||||
total_view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
banner_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
external_links: Mapped[list | None] = mapped_column(JSON)
|
||||
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
||||
default_language: Mapped[str | None] = mapped_column(String(16))
|
||||
country: Mapped[str | None] = mapped_column(String(8))
|
||||
# Creator keyword tags (brandingSettings.channel.keywords) — a single space-separated string,
|
||||
# multi-word tags quoted; the client parses it into chips for the About tab.
|
||||
keywords: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
# Sync bookkeeping.
|
||||
details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# When the newest uploads were last read through the Data API because the RSS feed would not
|
||||
# answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock.
|
||||
feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
||||
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this channel only entered the catalog because a live YouTube search
|
||||
# surfaced one of its videos (nobody subscribes to it / it has no organic upload sync).
|
||||
# Lets the Library hide search-only "noise" by default.
|
||||
from_search: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
# True when this channel exists only because a user EXPLORED it (opened its page without
|
||||
# subscribing) — an ephemeral, per-explorer addition. The cleanup job hard-deletes such a
|
||||
# channel (and its via_explore videos) once nobody keeps it (no subscription, no live
|
||||
# ExploredChannel row, no per-user state) after a grace period. Cleared when someone
|
||||
# subscribes (the "keep" signal promotes it to organic).
|
||||
from_explore: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
|
||||
videos: Mapped[list["Video"]] = relationship(back_populates="channel")
|
||||
subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel")
|
||||
|
||||
|
||||
class Subscription(Base, TimestampMixin):
|
||||
"""Per-user link to a channel, with the user's own priority/visibility overrides."""
|
||||
|
||||
__tablename__ = "subscriptions"
|
||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# YouTube's subscription resource id, needed to unsubscribe via the API.
|
||||
yt_subscription_id: Mapped[str | None] = mapped_column(String(128))
|
||||
priority: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
hidden: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Per-user opt-in to full-history (deep) backfill of this channel. Default off so a
|
||||
# new user's unique channels don't trigger a big shared-quota burst; the deep
|
||||
# scheduler only picks up channels at least one user has requested.
|
||||
deep_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
subscribed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
user: Mapped["User"] = relationship(back_populates="subscriptions")
|
||||
channel: Mapped["Channel"] = relationship(back_populates="subscriptions")
|
||||
|
||||
|
||||
# Video.live_status values that mean "currently a live/upcoming broadcast" — transient states
|
||||
# hidden from feeds/search and revisited by the refresh job until they settle into a VOD.
|
||||
LIVE_OR_UPCOMING = ("live", "upcoming")
|
||||
|
||||
|
||||
class Video(Base, TimestampMixin):
|
||||
"""A YouTube video. Shared across users; per-user state lives elsewhere."""
|
||||
|
||||
__tablename__ = "videos"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
||||
# No standalone channel_id index: the composite `ix_videos_channel_pub_cover`
|
||||
# (channel_id, published_at DESC NULLS LAST, id) from migration 0095 leads with channel_id, so it
|
||||
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) and the per-channel feed
|
||||
# sort. Its INCLUDEd columns make the channel manager's sums and the feed sidebar's counts Index Only
|
||||
# Scans (no heap): a column those queries start reading belongs in it (test_channel_list_index_only).
|
||||
# Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
||||
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
||||
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
# NOT NULL (migration 0068): in reality every video carries a publish date, and a nullable key
|
||||
# forced the feed's keyset predicate into a non-sargable `... OR published_at IS NULL` disjunction
|
||||
# that blocked the ix_videos_published_at_desc range (deep pages scanned from the top and filtered —
|
||||
# PERF-15). server_default now() is the safety net for the rare date-less ingest (the enrich fills
|
||||
# the real date); the ix_videos_published_at index stays for the `oldest` sort.
|
||||
published_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), index=True, nullable=False, server_default=func.now()
|
||||
)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
|
||||
# ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to
|
||||
# `feedquery.order_by` by test_feed_sort_indexes.
|
||||
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
category_id: Mapped[int | None] = mapped_column(Integer)
|
||||
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
||||
default_language: Mapped[str | None] = mapped_column(String(16))
|
||||
detected_language: Mapped[str | None] = mapped_column(String(16))
|
||||
# Creator-supplied keyword tags (snippet.tags joined by spaces) — fetched for free with the
|
||||
# snippet we already request. Indexed into search_vector so the feed search matches the
|
||||
# uploader's own keywords, not just words in the title.
|
||||
keywords: Mapped[str | None] = mapped_column(Text)
|
||||
# Accumulated distinct live-search queries that surfaced this video (across all users) —
|
||||
# folds YouTube's own relevance judgement into the local search: a video YouTube returned
|
||||
# for "ti amo magyarul" becomes findable by that query locally even if its title lacks it.
|
||||
search_terms: Mapped[str | None] = mapped_column(Text)
|
||||
# No btree here or on the flag columns below: migration 0069 (PERF-3) dropped the never-picked
|
||||
# boolean btrees and replaced the useful ones with partial indexes over the target subset
|
||||
# (via_search=true, NOT shorts_probed, unavailable_since IS NOT NULL) — migration-defined, like the
|
||||
# search_vector GIN.
|
||||
is_short: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Whether the youtube.com/shorts/<id> probe has run for this video.
|
||||
shorts_probed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this video first entered the catalog via a live YouTube search (not an
|
||||
# organic channel/playlist sync). Used to hide search-discovered "noise" from the
|
||||
# Library by default; left False by the organic ingest paths.
|
||||
via_search: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this video entered the catalog because a user EXPLORED its (un-subscribed)
|
||||
# channel. Like via_search it's hidden from the shared Library by default, but it's
|
||||
# additionally PER-USER private: a via_explore video is only visible to users who
|
||||
# explored or subscribe to its channel (a gate in the feed query), so one user's
|
||||
# curiosity never floods everyone else's catalog. Left False by the organic ingest paths.
|
||||
via_explore: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# none | live | upcoming | premiere | was_live
|
||||
live_status: Mapped[str] = mapped_column(
|
||||
String(16), default="none", server_default="none", index=True
|
||||
)
|
||||
enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# videos.list?part=status fields (populated during enrichment); used by the
|
||||
# maintenance/validation job to decide whether a video is playable anywhere.
|
||||
embeddable: Mapped[bool | None] = mapped_column(Boolean)
|
||||
privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private
|
||||
upload_status: Mapped[str | None] = mapped_column(
|
||||
String(16)
|
||||
) # processed|failed|rejected|deleted
|
||||
# Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation
|
||||
# (least-recently-checked first); unavailable_since marks a video as currently
|
||||
# unplayable (hidden from the feed immediately, hard-deleted after a grace period);
|
||||
# unavailable_reason records why (deleted|private|paywalled|abandoned|stuck_live).
|
||||
# last_checked_at: no plain btree — migration 0069 (PERF-18) replaced it with the partial
|
||||
# ix_videos_recheck_order (last_checked_at NULLS FIRST WHERE unavailable_since IS NULL) that the
|
||||
# revalidate job's exact ORDER BY rides; unavailable_since is a partial WHERE IS NOT NULL (PERF-3).
|
||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
|
||||
|
||||
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
|
||||
# creator keywords + search queries (B) > truncated description (C). ts_rank honours the
|
||||
# weights so title matches outrank description matches. Backed by a GIN index (migration
|
||||
# 0032). The accent-insensitive `unaccent_simple` config comes from migration 0031.
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
TSVECTOR,
|
||||
Computed(
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(title, '')), 'A') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(keywords, '') || ' ' || "
|
||||
"coalesce(search_terms, '')), 'B') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(description, ''), 1000)), 'C')",
|
||||
persisted=True,
|
||||
),
|
||||
)
|
||||
|
||||
channel: Mapped["Channel"] = relationship(back_populates="videos")
|
||||
|
||||
|
||||
class SearchFind(Base, TimestampMixin):
|
||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||
|
||||
Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source
|
||||
filter — distinct from the GLOBAL `Video.via_search` provenance, which marks a video as
|
||||
search-discovered by *anyone* and drives the shared Library's Source filter."""
|
||||
|
||||
__tablename__ = "search_finds"
|
||||
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
|
||||
class ExploredChannel(Base, TimestampMixin):
|
||||
"""Per-user record of a channel the user opened the page of WITHOUT subscribing.
|
||||
|
||||
Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the
|
||||
user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the
|
||||
explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period
|
||||
purge: a from_explore channel with no live ExploredChannel row (and no subscription / no
|
||||
per-user state on its videos) is hard-deleted by the cleanup job. Subscribing is the "keep"
|
||||
signal — it promotes the channel to organic and these rows become irrelevant. `created_at`
|
||||
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
|
||||
|
||||
__tablename__ = "explored_channels"
|
||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class BlockedChannel(Base, TimestampMixin):
|
||||
"""Per-user channel blocklist. Videos from a blocked channel are excluded from this user's
|
||||
live YouTube search (and not ingested on their behalf) and hidden from their feed / explore
|
||||
views — a personal "never show me this channel again" that doesn't affect other users."""
|
||||
|
||||
__tablename__ = "blocked_channels"
|
||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class ApiQuotaUsage(Base):
|
||||
"""Tracks YouTube Data API units spent per Pacific-time day (the quota resets at
|
||||
midnight Pacific). The whole app shares one daily budget."""
|
||||
|
||||
Reference in New Issue
Block a user