refactor(catalog): the unit's tables move into catalog/models
Channel, Subscription, Video, SearchFind, ExploredChannel and BlockedChannel (and LIVE_OR_UPCOMING) leave app/models.py for app/catalog/models.py; app.tables registers them. The User↔Subscription relationship names its other side as a type only. No schema change.
This commit is contained in:
@@ -0,0 +1,265 @@
|
||||
"""Catalog's tables: the shared YouTube world every user reads — channels and their videos — and
|
||||
each user's footprint in it: what they subscribe to, block, explored or found by search. Only this
|
||||
unit writes them; another unit reads them along a declared edge, or asks this unit's service."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
Computed,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.dialects.postgresql import TSVECTOR
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from app.db import Base
|
||||
from app.models import TimestampMixin
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.models import User
|
||||
|
||||
|
||||
class Channel(Base, TimestampMixin):
|
||||
"""A YouTube channel. Shared across all users (one channel's videos are the same
|
||||
for everyone), so its expensive metadata is fetched and stored only once."""
|
||||
|
||||
__tablename__ = "channels"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True) # "UC..." channel id
|
||||
title: Mapped[str | None] = mapped_column(String(255))
|
||||
handle: Mapped[str | None] = mapped_column(String(255))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
uploads_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
||||
subscriber_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
video_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
# Channel "About" metadata (fetched with the same channels.list call, no extra quota):
|
||||
# lifetime view count, the channel's join date, banner image, and external links.
|
||||
total_view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
banner_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
external_links: Mapped[list | None] = mapped_column(JSON)
|
||||
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
||||
default_language: Mapped[str | None] = mapped_column(String(16))
|
||||
country: Mapped[str | None] = mapped_column(String(8))
|
||||
# Creator keyword tags (brandingSettings.channel.keywords) — a single space-separated string,
|
||||
# multi-word tags quoted; the client parses it into chips for the About tab.
|
||||
keywords: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
# Sync bookkeeping.
|
||||
details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# When the newest uploads were last read through the Data API because the RSS feed would not
|
||||
# answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock.
|
||||
feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
||||
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this channel only entered the catalog because a live YouTube search
|
||||
# surfaced one of its videos (nobody subscribes to it / it has no organic upload sync).
|
||||
# Lets the Library hide search-only "noise" by default.
|
||||
from_search: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
# True when this channel exists only because a user EXPLORED it (opened its page without
|
||||
# subscribing) — an ephemeral, per-explorer addition. The cleanup job hard-deletes such a
|
||||
# channel (and its via_explore videos) once nobody keeps it (no subscription, no live
|
||||
# ExploredChannel row, no per-user state) after a grace period. Cleared when someone
|
||||
# subscribes (the "keep" signal promotes it to organic).
|
||||
from_explore: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
|
||||
videos: Mapped[list["Video"]] = relationship(back_populates="channel")
|
||||
subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel")
|
||||
|
||||
|
||||
class Subscription(Base, TimestampMixin):
|
||||
"""Per-user link to a channel, with the user's own priority/visibility overrides."""
|
||||
|
||||
__tablename__ = "subscriptions"
|
||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# YouTube's subscription resource id, needed to unsubscribe via the API.
|
||||
yt_subscription_id: Mapped[str | None] = mapped_column(String(128))
|
||||
priority: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
hidden: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Per-user opt-in to full-history (deep) backfill of this channel. Default off so a
|
||||
# new user's unique channels don't trigger a big shared-quota burst; the deep
|
||||
# scheduler only picks up channels at least one user has requested.
|
||||
deep_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
subscribed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
user: Mapped["User"] = relationship(back_populates="subscriptions")
|
||||
channel: Mapped["Channel"] = relationship(back_populates="subscriptions")
|
||||
|
||||
|
||||
# Video.live_status values that mean "currently a live/upcoming broadcast" — transient states
|
||||
# hidden from feeds/search and revisited by the refresh job until they settle into a VOD.
|
||||
LIVE_OR_UPCOMING = ("live", "upcoming")
|
||||
|
||||
|
||||
class Video(Base, TimestampMixin):
|
||||
"""A YouTube video. Shared across users; per-user state lives elsewhere."""
|
||||
|
||||
__tablename__ = "videos"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
||||
# No standalone channel_id index: the composite `ix_videos_channel_pub_cover`
|
||||
# (channel_id, published_at DESC NULLS LAST, id) from migration 0095 leads with channel_id, so it
|
||||
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) and the per-channel feed
|
||||
# sort. Its INCLUDEd columns make the channel manager's sums and the feed sidebar's counts Index Only
|
||||
# Scans (no heap): a column those queries start reading belongs in it (test_channel_list_index_only).
|
||||
# Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
||||
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
||||
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
# NOT NULL (migration 0068): in reality every video carries a publish date, and a nullable key
|
||||
# forced the feed's keyset predicate into a non-sargable `... OR published_at IS NULL` disjunction
|
||||
# that blocked the ix_videos_published_at_desc range (deep pages scanned from the top and filtered —
|
||||
# PERF-15). server_default now() is the safety net for the rare date-less ingest (the enrich fills
|
||||
# the real date); the ix_videos_published_at index stays for the `oldest` sort.
|
||||
published_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), index=True, nullable=False, server_default=func.now()
|
||||
)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
|
||||
# ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to
|
||||
# `feedquery.order_by` by test_feed_sort_indexes.
|
||||
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
category_id: Mapped[int | None] = mapped_column(Integer)
|
||||
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
||||
default_language: Mapped[str | None] = mapped_column(String(16))
|
||||
detected_language: Mapped[str | None] = mapped_column(String(16))
|
||||
# Creator-supplied keyword tags (snippet.tags joined by spaces) — fetched for free with the
|
||||
# snippet we already request. Indexed into search_vector so the feed search matches the
|
||||
# uploader's own keywords, not just words in the title.
|
||||
keywords: Mapped[str | None] = mapped_column(Text)
|
||||
# Accumulated distinct live-search queries that surfaced this video (across all users) —
|
||||
# folds YouTube's own relevance judgement into the local search: a video YouTube returned
|
||||
# for "ti amo magyarul" becomes findable by that query locally even if its title lacks it.
|
||||
search_terms: Mapped[str | None] = mapped_column(Text)
|
||||
# No btree here or on the flag columns below: migration 0069 (PERF-3) dropped the never-picked
|
||||
# boolean btrees and replaced the useful ones with partial indexes over the target subset
|
||||
# (via_search=true, NOT shorts_probed, unavailable_since IS NOT NULL) — migration-defined, like the
|
||||
# search_vector GIN.
|
||||
is_short: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Whether the youtube.com/shorts/<id> probe has run for this video.
|
||||
shorts_probed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this video first entered the catalog via a live YouTube search (not an
|
||||
# organic channel/playlist sync). Used to hide search-discovered "noise" from the
|
||||
# Library by default; left False by the organic ingest paths.
|
||||
via_search: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this video entered the catalog because a user EXPLORED its (un-subscribed)
|
||||
# channel. Like via_search it's hidden from the shared Library by default, but it's
|
||||
# additionally PER-USER private: a via_explore video is only visible to users who
|
||||
# explored or subscribe to its channel (a gate in the feed query), so one user's
|
||||
# curiosity never floods everyone else's catalog. Left False by the organic ingest paths.
|
||||
via_explore: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# none | live | upcoming | premiere | was_live
|
||||
live_status: Mapped[str] = mapped_column(
|
||||
String(16), default="none", server_default="none", index=True
|
||||
)
|
||||
enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# videos.list?part=status fields (populated during enrichment); used by the
|
||||
# maintenance/validation job to decide whether a video is playable anywhere.
|
||||
embeddable: Mapped[bool | None] = mapped_column(Boolean)
|
||||
privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private
|
||||
upload_status: Mapped[str | None] = mapped_column(
|
||||
String(16)
|
||||
) # processed|failed|rejected|deleted
|
||||
# Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation
|
||||
# (least-recently-checked first); unavailable_since marks a video as currently
|
||||
# unplayable (hidden from the feed immediately, hard-deleted after a grace period);
|
||||
# unavailable_reason records why (deleted|private|paywalled|abandoned|stuck_live).
|
||||
# last_checked_at: no plain btree — migration 0069 (PERF-18) replaced it with the partial
|
||||
# ix_videos_recheck_order (last_checked_at NULLS FIRST WHERE unavailable_since IS NULL) that the
|
||||
# revalidate job's exact ORDER BY rides; unavailable_since is a partial WHERE IS NOT NULL (PERF-3).
|
||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
|
||||
|
||||
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
|
||||
# creator keywords + search queries (B) > truncated description (C). ts_rank honours the
|
||||
# weights so title matches outrank description matches. Backed by a GIN index (migration
|
||||
# 0032). The accent-insensitive `unaccent_simple` config comes from migration 0031.
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
TSVECTOR,
|
||||
Computed(
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(title, '')), 'A') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(keywords, '') || ' ' || "
|
||||
"coalesce(search_terms, '')), 'B') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(description, ''), 1000)), 'C')",
|
||||
persisted=True,
|
||||
),
|
||||
)
|
||||
|
||||
channel: Mapped["Channel"] = relationship(back_populates="videos")
|
||||
|
||||
|
||||
class SearchFind(Base, TimestampMixin):
|
||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||
|
||||
Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source
|
||||
filter — distinct from the GLOBAL `Video.via_search` provenance, which marks a video as
|
||||
search-discovered by *anyone* and drives the shared Library's Source filter."""
|
||||
|
||||
__tablename__ = "search_finds"
|
||||
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
|
||||
class ExploredChannel(Base, TimestampMixin):
|
||||
"""Per-user record of a channel the user opened the page of WITHOUT subscribing.
|
||||
|
||||
Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the
|
||||
user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the
|
||||
explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period
|
||||
purge: a from_explore channel with no live ExploredChannel row (and no subscription / no
|
||||
per-user state on its videos) is hard-deleted by the cleanup job. Subscribing is the "keep"
|
||||
signal — it promotes the channel to organic and these rows become irrelevant. `created_at`
|
||||
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
|
||||
|
||||
__tablename__ = "explored_channels"
|
||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class BlockedChannel(Base, TimestampMixin):
|
||||
"""Per-user channel blocklist. Videos from a blocked channel are excluded from this user's
|
||||
live YouTube search (and not ingested on their behalf) and hidden from their feed / explore
|
||||
views — a personal "never show me this channel again" that doesn't affect other users."""
|
||||
|
||||
__tablename__ = "blocked_channels"
|
||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
@@ -22,7 +22,7 @@ to the client, which has both the native payload and the translations.
|
||||
"""
|
||||
|
||||
from app.discover.budget import remaining_seconds
|
||||
from app.models import Channel
|
||||
from app.catalog.models import Channel
|
||||
from app.reader.models import Playlist
|
||||
|
||||
# Plex artwork is a poster for the things you browse and a landscape still for the things you play.
|
||||
|
||||
@@ -22,7 +22,8 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.discover import cards, queries
|
||||
from app.models import Channel, PlexItem, PlexShow, User, Video
|
||||
from app.models import PlexItem, PlexShow, User
|
||||
from app.catalog.models import Channel, Video
|
||||
from app.reader.models import ComfortPin, Playlist
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import SHOW_NEW, leaf_card, show_card
|
||||
|
||||
@@ -29,7 +29,8 @@ from app.feedquery import (
|
||||
serialize,
|
||||
sort_keys,
|
||||
)
|
||||
from app.models import Channel, PlexItem, PlexShow, PlexState, User, Video
|
||||
from app.models import PlexItem, PlexShow, PlexState, User
|
||||
from app.catalog.models import Channel, Video
|
||||
from app.reader.models import PlaylistItem, VideoState
|
||||
from app.plex import rollup
|
||||
from app.plex.cards import display_title, leaf_card, movie_card, show_card
|
||||
|
||||
@@ -24,7 +24,8 @@ from app.downloads import credentials
|
||||
from app.downloads import edit as editmod
|
||||
from app.downloads import folders, formats, mediakind, quota, storage
|
||||
from app.downloads.status import HOLDING, JobKind, JobStatus
|
||||
from app.models import DISPLAY_TEXT_MAX, Channel, PlexItem, PlexShow, User, Video
|
||||
from app.models import DISPLAY_TEXT_MAX, PlexItem, PlexShow, User
|
||||
from app.catalog.models import Channel, Video
|
||||
from app.downloads.models import (
|
||||
DownloadJob,
|
||||
DownloadProfile,
|
||||
|
||||
@@ -17,14 +17,14 @@ from sqlalchemy import BigInteger, Select, and_, case, cast, false, func, or_, s
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from app.collation import collated
|
||||
from app.models import (
|
||||
from app.models import User
|
||||
from app.catalog.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
ExploredChannel,
|
||||
SearchFind,
|
||||
Subscription,
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, Tag, VideoState
|
||||
|
||||
+4
-237
@@ -1,4 +1,5 @@
|
||||
from datetime import date, datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
@@ -26,6 +27,9 @@ from app.db import Base
|
||||
from app.runstatus import RunStatus, RunTrigger, sql_in
|
||||
from app.utils import canonical_email
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.catalog.models import Subscription
|
||||
|
||||
# Every user-supplied display string in this schema shares one width, and it lives HERE so the
|
||||
# columns below are declared with it rather than with a literal: a name a person typed goes into
|
||||
# `User.display_name` (their Google profile name), `DownloadJob.display_name` and
|
||||
@@ -220,243 +224,6 @@ class Invite(Base, CanonicalEmailMixin):
|
||||
lang: Mapped[str | None] = mapped_column(String(8))
|
||||
|
||||
|
||||
class Channel(Base, TimestampMixin):
|
||||
"""A YouTube channel. Shared across all users (one channel's videos are the same
|
||||
for everyone), so its expensive metadata is fetched and stored only once."""
|
||||
|
||||
__tablename__ = "channels"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(32), primary_key=True) # "UC..." channel id
|
||||
title: Mapped[str | None] = mapped_column(String(255))
|
||||
handle: Mapped[str | None] = mapped_column(String(255))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
uploads_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
||||
subscriber_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
video_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
# Channel "About" metadata (fetched with the same channels.list call, no extra quota):
|
||||
# lifetime view count, the channel's join date, banner image, and external links.
|
||||
total_view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
banner_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
external_links: Mapped[list | None] = mapped_column(JSON)
|
||||
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
||||
default_language: Mapped[str | None] = mapped_column(String(16))
|
||||
country: Mapped[str | None] = mapped_column(String(8))
|
||||
# Creator keyword tags (brandingSettings.channel.keywords) — a single space-separated string,
|
||||
# multi-word tags quoted; the client parses it into chips for the About tab.
|
||||
keywords: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
# Sync bookkeeping.
|
||||
details_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
recent_synced_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
last_rss_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# When the newest uploads were last read through the Data API because the RSS feed would not
|
||||
# answer (`sync/runner.py` `run_feed_fallback`): the once-per-`rss_fallback_hours` clock.
|
||||
feed_fallback_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
backfill_cursor: Mapped[str | None] = mapped_column(String(128))
|
||||
backfill_done: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this channel only entered the catalog because a live YouTube search
|
||||
# surfaced one of its videos (nobody subscribes to it / it has no organic upload sync).
|
||||
# Lets the Library hide search-only "noise" by default.
|
||||
from_search: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
# True when this channel exists only because a user EXPLORED it (opened its page without
|
||||
# subscribing) — an ephemeral, per-explorer addition. The cleanup job hard-deletes such a
|
||||
# channel (and its via_explore videos) once nobody keeps it (no subscription, no live
|
||||
# ExploredChannel row, no per-user state) after a grace period. Cleared when someone
|
||||
# subscribes (the "keep" signal promotes it to organic).
|
||||
from_explore: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
|
||||
videos: Mapped[list["Video"]] = relationship(back_populates="channel")
|
||||
subscriptions: Mapped[list["Subscription"]] = relationship(back_populates="channel")
|
||||
|
||||
|
||||
class Subscription(Base, TimestampMixin):
|
||||
"""Per-user link to a channel, with the user's own priority/visibility overrides."""
|
||||
|
||||
__tablename__ = "subscriptions"
|
||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_channel"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
# YouTube's subscription resource id, needed to unsubscribe via the API.
|
||||
yt_subscription_id: Mapped[str | None] = mapped_column(String(128))
|
||||
priority: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
||||
hidden: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Per-user opt-in to full-history (deep) backfill of this channel. Default off so a
|
||||
# new user's unique channels don't trigger a big shared-quota burst; the deep
|
||||
# scheduler only picks up channels at least one user has requested.
|
||||
deep_requested: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
subscribed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
user: Mapped["User"] = relationship(back_populates="subscriptions")
|
||||
channel: Mapped["Channel"] = relationship(back_populates="subscriptions")
|
||||
|
||||
|
||||
# Video.live_status values that mean "currently a live/upcoming broadcast" — transient states
|
||||
# hidden from feeds/search and revisited by the refresh job until they settle into a VOD.
|
||||
LIVE_OR_UPCOMING = ("live", "upcoming")
|
||||
|
||||
|
||||
class Video(Base, TimestampMixin):
|
||||
"""A YouTube video. Shared across users; per-user state lives elsewhere."""
|
||||
|
||||
__tablename__ = "videos"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(16), primary_key=True) # video id
|
||||
# No standalone channel_id index: the composite `ix_videos_channel_pub_cover`
|
||||
# (channel_id, published_at DESC NULLS LAST, id) from migration 0095 leads with channel_id, so it
|
||||
# serves every channel_id lookup (FK cascade, sync upsert, channel-detail) and the per-channel feed
|
||||
# sort. Its INCLUDEd columns make the channel manager's sums and the feed sidebar's counts Index Only
|
||||
# Scans (no heap): a column those queries start reading belongs in it (test_channel_list_index_only).
|
||||
# Like the search_vector GIN, this composite is migration-defined, not an inline `index=True`.
|
||||
channel_id: Mapped[str] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
||||
title: Mapped[str | None] = mapped_column(Text) # normalized for display (see app.titles)
|
||||
original_title: Mapped[str | None] = mapped_column(Text) # raw YouTube title, preserved
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
# NOT NULL (migration 0068): in reality every video carries a publish date, and a nullable key
|
||||
# forced the feed's keyset predicate into a non-sargable `... OR published_at IS NULL` disjunction
|
||||
# that blocked the ix_videos_published_at_desc range (deep pages scanned from the top and filtered —
|
||||
# PERF-15). server_default now() is the safety net for the rare date-less ingest (the enrich fills
|
||||
# the real date); the ix_videos_published_at index stays for the `oldest` sort.
|
||||
published_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), index=True, nullable=False, server_default=func.now()
|
||||
)
|
||||
thumbnail_url: Mapped[str | None] = mapped_column(String(1024))
|
||||
# The feed's views / duration / title sorts each ride a migration-defined index in their exact
|
||||
# ORDER BY, one per direction (and per ICU collation for title): migration 0096, held to
|
||||
# `feedquery.order_by` by test_feed_sort_indexes.
|
||||
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||
view_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
like_count: Mapped[int | None] = mapped_column(BigInteger)
|
||||
category_id: Mapped[int | None] = mapped_column(Integer)
|
||||
topic_categories: Mapped[list | None] = mapped_column(JSON)
|
||||
default_language: Mapped[str | None] = mapped_column(String(16))
|
||||
detected_language: Mapped[str | None] = mapped_column(String(16))
|
||||
# Creator-supplied keyword tags (snippet.tags joined by spaces) — fetched for free with the
|
||||
# snippet we already request. Indexed into search_vector so the feed search matches the
|
||||
# uploader's own keywords, not just words in the title.
|
||||
keywords: Mapped[str | None] = mapped_column(Text)
|
||||
# Accumulated distinct live-search queries that surfaced this video (across all users) —
|
||||
# folds YouTube's own relevance judgement into the local search: a video YouTube returned
|
||||
# for "ti amo magyarul" becomes findable by that query locally even if its title lacks it.
|
||||
search_terms: Mapped[str | None] = mapped_column(Text)
|
||||
# No btree here or on the flag columns below: migration 0069 (PERF-3) dropped the never-picked
|
||||
# boolean btrees and replaced the useful ones with partial indexes over the target subset
|
||||
# (via_search=true, NOT shorts_probed, unavailable_since IS NOT NULL) — migration-defined, like the
|
||||
# search_vector GIN.
|
||||
is_short: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# Whether the youtube.com/shorts/<id> probe has run for this video.
|
||||
shorts_probed: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this video first entered the catalog via a live YouTube search (not an
|
||||
# organic channel/playlist sync). Used to hide search-discovered "noise" from the
|
||||
# Library by default; left False by the organic ingest paths.
|
||||
via_search: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# True when this video entered the catalog because a user EXPLORED its (un-subscribed)
|
||||
# channel. Like via_search it's hidden from the shared Library by default, but it's
|
||||
# additionally PER-USER private: a via_explore video is only visible to users who
|
||||
# explored or subscribe to its channel (a gate in the feed query), so one user's
|
||||
# curiosity never floods everyone else's catalog. Left False by the organic ingest paths.
|
||||
via_explore: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
||||
# none | live | upcoming | premiere | was_live
|
||||
live_status: Mapped[str] = mapped_column(
|
||||
String(16), default="none", server_default="none", index=True
|
||||
)
|
||||
enriched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
# videos.list?part=status fields (populated during enrichment); used by the
|
||||
# maintenance/validation job to decide whether a video is playable anywhere.
|
||||
embeddable: Mapped[bool | None] = mapped_column(Boolean)
|
||||
privacy_status: Mapped[str | None] = mapped_column(String(16)) # public|unlisted|private
|
||||
upload_status: Mapped[str | None] = mapped_column(
|
||||
String(16)
|
||||
) # processed|failed|rejected|deleted
|
||||
# Maintenance/validation lifecycle. last_checked_at drives the rolling re-validation
|
||||
# (least-recently-checked first); unavailable_since marks a video as currently
|
||||
# unplayable (hidden from the feed immediately, hard-deleted after a grace period);
|
||||
# unavailable_reason records why (deleted|private|paywalled|abandoned|stuck_live).
|
||||
# last_checked_at: no plain btree — migration 0069 (PERF-18) replaced it with the partial
|
||||
# ix_videos_recheck_order (last_checked_at NULLS FIRST WHERE unavailable_since IS NULL) that the
|
||||
# revalidate job's exact ORDER BY rides; unavailable_since is a partial WHERE IS NOT NULL (PERF-3).
|
||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
unavailable_since: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
unavailable_reason: Mapped[str | None] = mapped_column(String(24))
|
||||
|
||||
# Weighted full-text search document (DB-generated, never written by the app): title (A) >
|
||||
# creator keywords + search queries (B) > truncated description (C). ts_rank honours the
|
||||
# weights so title matches outrank description matches. Backed by a GIN index (migration
|
||||
# 0032). The accent-insensitive `unaccent_simple` config comes from migration 0031.
|
||||
search_vector: Mapped[object | None] = mapped_column(
|
||||
TSVECTOR,
|
||||
Computed(
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(title, '')), 'A') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', coalesce(keywords, '') || ' ' || "
|
||||
"coalesce(search_terms, '')), 'B') || "
|
||||
"setweight(to_tsvector('public.unaccent_simple', left(coalesce(description, ''), 1000)), 'C')",
|
||||
persisted=True,
|
||||
),
|
||||
)
|
||||
|
||||
channel: Mapped["Channel"] = relationship(back_populates="videos")
|
||||
|
||||
|
||||
class SearchFind(Base, TimestampMixin):
|
||||
"""Per-user record of a video the user surfaced via their own live YouTube search.
|
||||
|
||||
Lets "videos I searched for" appear in that user's own (Mine) feed and the Mine Source
|
||||
filter — distinct from the GLOBAL `Video.via_search` provenance, which marks a video as
|
||||
search-discovered by *anyone* and drives the shared Library's Source filter."""
|
||||
|
||||
__tablename__ = "search_finds"
|
||||
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_search_video"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||
|
||||
|
||||
class ExploredChannel(Base, TimestampMixin):
|
||||
"""Per-user record of a channel the user opened the page of WITHOUT subscribing.
|
||||
|
||||
Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the
|
||||
user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the
|
||||
explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period
|
||||
purge: a from_explore channel with no live ExploredChannel row (and no subscription / no
|
||||
per-user state on its videos) is hard-deleted by the cleanup job. Subscribing is the "keep"
|
||||
signal — it promotes the channel to organic and these rows become irrelevant. `created_at`
|
||||
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
|
||||
|
||||
__tablename__ = "explored_channels"
|
||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_explored_channel"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class BlockedChannel(Base, TimestampMixin):
|
||||
"""Per-user channel blocklist. Videos from a blocked channel are excluded from this user's
|
||||
live YouTube search (and not ingested on their behalf) and hidden from their feed / explore
|
||||
views — a personal "never show me this channel again" that doesn't affect other users."""
|
||||
|
||||
__tablename__ = "blocked_channels"
|
||||
__table_args__ = (UniqueConstraint("user_id", "channel_id", name="uq_user_blocked_channel"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class ApiQuotaUsage(Base):
|
||||
"""Tracks YouTube Data API units spent per Pacific-time day (the quota resets at
|
||||
midnight Pacific). The whole app shares one daily budget."""
|
||||
|
||||
@@ -11,7 +11,8 @@ from sqlalchemy import ColumnElement, and_, func, or_, select
|
||||
from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
|
||||
|
||||
from app.feedquery import feed_columns
|
||||
from app.models import Channel, User, Video
|
||||
from app.models import User
|
||||
from app.catalog.models import Channel, Video
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
|
||||
@@ -15,13 +15,13 @@ from app.mailtext import lang_for_user
|
||||
from app import logs, quota, scheduler
|
||||
from app.auth import admin_user, current_user, require_write_scope
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
from app.models import User
|
||||
from app.catalog.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
ExploredChannel,
|
||||
Subscription,
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, Tag
|
||||
|
||||
@@ -18,7 +18,8 @@ from sqlalchemy.orm import Session
|
||||
from app import scheduler
|
||||
from app.auth import admin_user
|
||||
from app.db import get_db
|
||||
from app.models import Channel, JobRun, LogEvent, User
|
||||
from app.models import JobRun, LogEvent, User
|
||||
from app.catalog.models import Channel
|
||||
from app.routes import _account as account
|
||||
from app.routes._keyset import PAGE_MAX, accounts, older_than, page, utc
|
||||
from app.runstatus import RunStatus
|
||||
|
||||
@@ -12,7 +12,8 @@ from app.audit import AuditAction
|
||||
from app.auth import admin_user
|
||||
from app.config import settings
|
||||
from app.db import get_db
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
from app.models import User
|
||||
from app.catalog.models import Channel, Subscription, Video
|
||||
from app.scheduler import (
|
||||
JOB_INTERVALS,
|
||||
MAX_INTERVAL,
|
||||
|
||||
@@ -25,13 +25,13 @@ from sqlalchemy.orm import Session, aliased
|
||||
from app import logs, quota, sysconfig
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.models import (
|
||||
from app.models import User
|
||||
from app.catalog.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
SearchFind,
|
||||
Subscription,
|
||||
User,
|
||||
Video,
|
||||
)
|
||||
from app.reader.models import Playlist, PlaylistItem, VideoState
|
||||
|
||||
@@ -6,7 +6,8 @@ from app import audit, quota, state
|
||||
from app.audit import AuditAction
|
||||
from app.auth import admin_user, current_user, has_read_scope
|
||||
from app.db import get_db
|
||||
from app.models import Channel, Subscription, User, Video
|
||||
from app.models import User
|
||||
from app.catalog.models import Channel, Subscription, Video
|
||||
from app.routes._common import youtube_refusal
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.sync.runner import estimate_deep_backfill
|
||||
|
||||
@@ -15,7 +15,8 @@ from app import audit, debugsession, jobruns, logs, logsink, progress, quota, sy
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.downloads.gc import run_download_gc
|
||||
from app.models import Channel, JobRun
|
||||
from app.models import JobRun
|
||||
from app.catalog.models import Channel
|
||||
from app.plex.sync import sync as run_plex_sync
|
||||
from app.plex.tmdb import run_enrich as run_tmdb_enrich
|
||||
from app.plex.tmdb import run_people_enrich
|
||||
|
||||
@@ -12,7 +12,7 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, progress, sysconfig
|
||||
from app.models import Channel, Video
|
||||
from app.catalog.models import Channel, Video
|
||||
from app.reader import queries as reader_queries
|
||||
from app.reader import service as reader_service
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.models import Channel, ExploredChannel, SearchFind, Subscription, Video
|
||||
from app.catalog.models import Channel, ExploredChannel, SearchFind, Subscription, Video
|
||||
from app.reader import queries as reader_queries
|
||||
from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
|
||||
from app.youtube.client import YouTubeClient, YouTubeError, is_playlist_gone
|
||||
|
||||
@@ -27,7 +27,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, progress, quota, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import Video
|
||||
from app.catalog.models import Video
|
||||
from app.reader import queries as reader_queries
|
||||
from app.notifications import create_notification
|
||||
from app.sync.runner import get_service_user
|
||||
|
||||
@@ -20,7 +20,8 @@ from sqlalchemy.pool import NullPool
|
||||
|
||||
from app import logs, progress, quota
|
||||
from app.auth import has_read_scope
|
||||
from app.models import User, Video
|
||||
from app.models import User
|
||||
from app.catalog.models import Video
|
||||
from app.reader.models import Playlist, PlaylistItem
|
||||
from app.sync.playlist_merge import merge_plan
|
||||
from app.sync.runner import token_users
|
||||
|
||||
@@ -16,7 +16,8 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, progress, quota, sysconfig
|
||||
from app.db import SessionLocal
|
||||
from app.models import Channel, OAuthToken, Subscription, User
|
||||
from app.models import OAuthToken, User
|
||||
from app.catalog.models import Channel, Subscription
|
||||
from app.sync.explore import followed_channel
|
||||
from app.sync.subscriptions import import_subscriptions
|
||||
from app.sync.videos import (
|
||||
|
||||
@@ -7,7 +7,8 @@ from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import Channel, Subscription, User
|
||||
from app.models import User
|
||||
from app.catalog.models import Channel, Subscription
|
||||
from app.utils import normalize_description
|
||||
from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail
|
||||
|
||||
|
||||
@@ -23,7 +23,8 @@ from collections import Counter
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Subscription, User
|
||||
from app.models import User
|
||||
from app.catalog.models import Subscription
|
||||
from app.reader.models import ChannelTag, Tag
|
||||
from app.userscope import owned_by
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, progress, quota, sysconfig
|
||||
from app.models import LIVE_OR_UPCOMING, Channel, Video
|
||||
from app.catalog.models import LIVE_OR_UPCOMING, Channel, Video
|
||||
from app.titles import normalize_title
|
||||
from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail, is_playlist_gone
|
||||
from app.youtube.shorts import make_client, probe_is_short
|
||||
|
||||
@@ -5,6 +5,7 @@ names the whole schema. Whatever needs all of it — the migrations, the test da
|
||||
every mapper — imports this module.
|
||||
"""
|
||||
|
||||
import app.catalog.models # Catalog's tables
|
||||
import app.downloads.models # Capture's tables
|
||||
import app.models # the core tables
|
||||
import app.reader.models # noqa: F401 — Reader's tables
|
||||
|
||||
Reference in New Issue
Block a user