refactor(catalog): the HTTP layer reads through queries and writes through the service
Channels, search and sync do no database work in their routes any more: reads go through catalog/queries, writes through catalog/service, and a live search's ingest is catalog/search. What a catalog route needs from Reader (a playlist's channels, a user's tag links, the feed projection of results) it asks reader/queries and reader/service. The scheduler and the runs page read channel titles and the work queue through catalog/queries. Catalog is fenced below its HTTP layer: its owner filters go through userscope, it reads no other unit's tables, and its ingest jobs commit only where they are named.
This commit is contained in:
@@ -0,0 +1,296 @@
|
||||
"""Catalog's reads: channels and videos, and each user's footprint in them (subscriptions, blocks,
|
||||
explorations), for the HTTP layer and for the core code that asks about them.
|
||||
|
||||
A user's row is read through `userscope`, so someone else's is the same answer as a missing one.
|
||||
Catalog does not read another unit's tables: what it needs from Reader (a playlist's channels, a
|
||||
user's tag links) it asks `reader.queries`.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import Select, and_, case, func, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.catalog.models import (
|
||||
LIVE_OR_UPCOMING,
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
ExploredChannel,
|
||||
Subscription,
|
||||
Video,
|
||||
)
|
||||
from app.collation import collated
|
||||
from app.models import User
|
||||
from app.sync.explore import followed_channel
|
||||
from app.userscope import owned_by
|
||||
|
||||
# --- channels --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def channel(db: Session, channel_id: str) -> Channel | None:
|
||||
return db.get(Channel, channel_id)
|
||||
|
||||
|
||||
def channel_title(db: Session, channel_id: str) -> str | None:
|
||||
return db.scalar(select(Channel.title).where(Channel.id == channel_id))
|
||||
|
||||
|
||||
def channel_titles(db: Session, ids: set[str]) -> dict[str, str]:
|
||||
"""Each channel's title by id; a channel with no title yet is left out."""
|
||||
if not ids:
|
||||
return {}
|
||||
rows = db.execute(select(Channel.id, Channel.title).where(Channel.id.in_(ids)))
|
||||
return {cid: title for cid, title in rows.tuples() if title is not None}
|
||||
|
||||
|
||||
def channel_aggregates(channel_ids: list[str]) -> Select:
|
||||
"""Per-channel stored count, last upload and total duration, in one grouped pass. Every column it
|
||||
reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index
|
||||
alone; a test holds that (test_channel_list_index_only)."""
|
||||
return (
|
||||
select(
|
||||
Video.channel_id,
|
||||
func.count(Video.id),
|
||||
func.max(Video.published_at),
|
||||
func.coalesce(func.sum(Video.duration_seconds), 0),
|
||||
)
|
||||
.where(Video.channel_id.in_(channel_ids))
|
||||
.group_by(Video.channel_id)
|
||||
)
|
||||
|
||||
|
||||
def aggregates(db: Session, channel_ids: list[str]) -> list[tuple[str, int, datetime | None, int]]:
|
||||
"""`channel_aggregates`, run: (channel id, stored videos, last upload, total seconds)."""
|
||||
if not channel_ids:
|
||||
return []
|
||||
return list(db.execute(channel_aggregates(channel_ids)).tuples())
|
||||
|
||||
|
||||
def video_counts(db: Session, channel_id: str) -> tuple[int, int, int]:
|
||||
"""A channel's stored videos, its Shorts and its live/upcoming ones, in one grouped pass."""
|
||||
n_all, n_short, n_live = db.execute(
|
||||
select(
|
||||
func.count(Video.id),
|
||||
func.count(Video.id).filter(Video.is_short.is_(True)),
|
||||
func.count(Video.id).filter(Video.live_status.in_(LIVE_OR_UPCOMING)),
|
||||
).where(Video.channel_id == channel_id)
|
||||
).one()
|
||||
return int(n_all or 0), int(n_short or 0), int(n_live or 0)
|
||||
|
||||
|
||||
def deep_queued(db: Session, channel_ids: list[str]) -> set[str]:
|
||||
"""Which of these channels at least one user (any user) has asked full history for."""
|
||||
if not channel_ids:
|
||||
return set()
|
||||
return set(
|
||||
db.execute(
|
||||
select(Subscription.channel_id)
|
||||
.where(
|
||||
Subscription.channel_id.in_(channel_ids),
|
||||
Subscription.deep_requested.is_(True),
|
||||
)
|
||||
.distinct()
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def followed(db: Session, channel_id: str) -> bool:
|
||||
"""Whether anyone subscribes to the channel."""
|
||||
return (
|
||||
db.execute(
|
||||
select(Subscription.id).where(Subscription.channel_id == channel_id).limit(1)
|
||||
).first()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
# --- a user's subscriptions, blocks and explorations ----------------------------------------
|
||||
|
||||
|
||||
def subscriptions(db: Session, user: User, lang: str) -> list[tuple[Subscription, Channel]]:
|
||||
"""The user's subscriptions with their channels, highest priority first, then by title in the
|
||||
reader's alphabet."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(Subscription, Channel)
|
||||
.join(Channel, Channel.id == Subscription.channel_id)
|
||||
.where(owned_by(Subscription.user_id, user))
|
||||
.order_by(Subscription.priority.desc(), collated(Channel.title, lang))
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def subscription(db: Session, user: User, channel_id: str) -> Subscription | None:
|
||||
return db.execute(
|
||||
select(Subscription).where(
|
||||
owned_by(Subscription.user_id, user), Subscription.channel_id == channel_id
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def subscribed_channels(user: User):
|
||||
"""The channel ids the user subscribes to, as a subquery."""
|
||||
return select(Subscription.channel_id).where(owned_by(Subscription.user_id, user))
|
||||
|
||||
|
||||
def exploration(db: Session, user: User, channel_id: str) -> ExploredChannel | None:
|
||||
return db.execute(
|
||||
select(ExploredChannel).where(
|
||||
owned_by(ExploredChannel.user_id, user), ExploredChannel.channel_id == channel_id
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def is_blocked(db: Session, user: User, channel_id: str) -> bool:
|
||||
return (
|
||||
db.execute(
|
||||
select(BlockedChannel.id).where(
|
||||
owned_by(BlockedChannel.user_id, user), BlockedChannel.channel_id == channel_id
|
||||
)
|
||||
).first()
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
def blocked_ids(db: Session, user: User) -> set[str]:
|
||||
return set(
|
||||
db.scalars(select(BlockedChannel.channel_id).where(owned_by(BlockedChannel.user_id, user)))
|
||||
)
|
||||
|
||||
|
||||
def blocked_channels(db: Session, user: User, lang: str) -> list[Channel]:
|
||||
return list(
|
||||
db.execute(
|
||||
select(Channel)
|
||||
.join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
|
||||
.where(owned_by(BlockedChannel.user_id, user))
|
||||
.order_by(collated(Channel.title, lang))
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
# --- sync progress ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def catalog_totals(db: Session, user: User) -> dict:
|
||||
"""The instance's catalog size and this user's subscription count (the sync status card)."""
|
||||
return {
|
||||
"subscriptions": db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Subscription)
|
||||
.where(owned_by(Subscription.user_id, user))
|
||||
),
|
||||
"channels_total": db.scalar(select(func.count()).select_from(Channel)),
|
||||
"channels_backfilling": db.scalar(
|
||||
select(func.count()).select_from(Channel).where(Channel.backfill_done.is_(False))
|
||||
),
|
||||
"videos_total": db.scalar(select(func.count()).select_from(Video)),
|
||||
"pending_enrich": db.scalar(
|
||||
select(func.count()).select_from(Video).where(Video.enriched_at.is_(None))
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def my_progress(db: Session, user: User) -> tuple[int, int, int, int]:
|
||||
"""Over the channels this user subscribes to: how many, how many have details, how many their
|
||||
recent uploads, how many their whole history."""
|
||||
total, details_synced, recent_synced, all_deep_done = db.execute(
|
||||
select(
|
||||
func.count(Channel.id),
|
||||
func.count(Channel.details_synced_at),
|
||||
func.count(Channel.recent_synced_at),
|
||||
func.sum(case((Channel.backfill_done.is_(True), 1), else_=0)),
|
||||
)
|
||||
.select_from(Channel)
|
||||
.join(
|
||||
Subscription,
|
||||
and_(Subscription.channel_id == Channel.id, owned_by(Subscription.user_id, user)),
|
||||
)
|
||||
).one()
|
||||
return total or 0, details_synced or 0, recent_synced or 0, int(all_deep_done or 0)
|
||||
|
||||
|
||||
def my_deep_requested(db: Session, user: User) -> list[Channel]:
|
||||
"""The channels this user asked full history for."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(Channel).join(
|
||||
Subscription,
|
||||
and_(
|
||||
Subscription.channel_id == Channel.id,
|
||||
owned_by(Subscription.user_id, user),
|
||||
Subscription.deep_requested.is_(True),
|
||||
),
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def my_video_count(db: Session, user: User) -> int:
|
||||
"""The videos of the channels this user subscribes to and has not hidden."""
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.count(Video.id)).join(
|
||||
Subscription,
|
||||
and_(
|
||||
Subscription.channel_id == Video.channel_id,
|
||||
owned_by(Subscription.user_id, user),
|
||||
Subscription.hidden.is_(False),
|
||||
),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
|
||||
|
||||
def video_total(db: Session) -> int:
|
||||
return db.scalar(select(func.count(Video.id))) or 0
|
||||
|
||||
|
||||
def deep_backlog(db: Session) -> list[Channel]:
|
||||
"""Every channel someone asked full history for that does not have it yet."""
|
||||
deep_requested = (
|
||||
select(Subscription.id)
|
||||
.where(Subscription.channel_id == Channel.id, Subscription.deep_requested.is_(True))
|
||||
.exists()
|
||||
)
|
||||
return list(
|
||||
db.execute(
|
||||
select(Channel).where(Channel.backfill_done.is_(False), deep_requested)
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def work_queue(db: Session) -> dict:
|
||||
"""What the catalog's jobs still have queued, instance-wide (the scheduler card)."""
|
||||
return {
|
||||
# The same selection run_recent_backfill works through, or the card counts channels the
|
||||
# job will never touch and never reaches zero.
|
||||
"channels_recent_pending": db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Channel)
|
||||
.where(Channel.recent_synced_at.is_(None), followed_channel())
|
||||
)
|
||||
or 0,
|
||||
"videos_pending_enrich": db.scalar(
|
||||
select(func.count()).select_from(Video).where(Video.enriched_at.is_(None))
|
||||
)
|
||||
or 0,
|
||||
"videos_pending_shorts": db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Video)
|
||||
.where(Video.shorts_probed.is_(False), Video.enriched_at.is_not(None))
|
||||
)
|
||||
or 0,
|
||||
"videos_live_refresh": db.scalar(
|
||||
select(func.count())
|
||||
.select_from(Video)
|
||||
.where(
|
||||
or_(
|
||||
Video.live_status.in_(("live", "upcoming")),
|
||||
and_(Video.live_status == "was_live", Video.duration_seconds.is_(None)),
|
||||
)
|
||||
)
|
||||
)
|
||||
or 0,
|
||||
}
|
||||
Reference in New Issue
Block a user