siftlode/backend/app/catalog/queries.py

"""Catalog's reads: channels and videos, and each user's footprint in them (subscriptions, blocks,
explorations), for the HTTP layer and for the core code that asks about them.

A user's row is read through `userscope`, so someone else's is the same answer as a missing one.
Catalog does not read another unit's tables: what it needs from Reader (a playlist's channels, a
user's tag links) it asks `reader.queries`.
"""

from datetime import datetime

from sqlalchemy import Select, and_, case, func, or_, select
from sqlalchemy.orm import Session

from app.catalog.models import (
    LIVE_OR_UPCOMING,
    BlockedChannel,
    Channel,
    ExploredChannel,
    Subscription,
    Video,
)
from app.collation import collated
from app.models import User
from app.sync.explore import followed_channel
from app.userscope import owned_by

# --- channels --------------------------------------------------------------------------------


def channel(db: Session, channel_id: str) -> Channel | None:
    return db.get(Channel, channel_id)


def channel_title(db: Session, channel_id: str) -> str | None:
    return db.scalar(select(Channel.title).where(Channel.id == channel_id))


def channel_titles(db: Session, ids: set[str]) -> dict[str, str]:
    """Each channel's title by id; a channel with no title yet is left out."""
    if not ids:
        return {}
    rows = db.execute(select(Channel.id, Channel.title).where(Channel.id.in_(ids)))
    return {cid: title for cid, title in rows.tuples() if title is not None}


def channel_aggregates(channel_ids: list[str]) -> Select:
    """Per-channel stored count, last upload and total duration, in one grouped pass. Every column it
    reads is in `ix_videos_channel_pub_cover` (migration 0095), so Postgres answers it from the index
    alone; a test holds that (test_channel_list_index_only)."""
    return (
        select(
            Video.channel_id,
            func.count(Video.id),
            func.max(Video.published_at),
            func.coalesce(func.sum(Video.duration_seconds), 0),
        )
        .where(Video.channel_id.in_(channel_ids))
        .group_by(Video.channel_id)
    )


def aggregates(db: Session, channel_ids: list[str]) -> list[tuple[str, int, datetime | None, int]]:
    """`channel_aggregates`, run: (channel id, stored videos, last upload, total seconds)."""
    if not channel_ids:
        return []
    return list(db.execute(channel_aggregates(channel_ids)).tuples())


def video_counts(db: Session, channel_id: str) -> tuple[int, int, int]:
    """A channel's stored videos, its Shorts and its live/upcoming ones, in one grouped pass."""
    n_all, n_short, n_live = db.execute(
        select(
            func.count(Video.id),
            func.count(Video.id).filter(Video.is_short.is_(True)),
            func.count(Video.id).filter(Video.live_status.in_(LIVE_OR_UPCOMING)),
        ).where(Video.channel_id == channel_id)
    ).one()
    return int(n_all or 0), int(n_short or 0), int(n_live or 0)


def deep_queued(db: Session, channel_ids: list[str]) -> set[str]:
    """Which of these channels at least one user (any user) has asked full history for."""
    if not channel_ids:
        return set()
    return set(
        db.execute(
            select(Subscription.channel_id)
            .where(
                Subscription.channel_id.in_(channel_ids),
                Subscription.deep_requested.is_(True),
            )
            .distinct()
        ).scalars()
    )


def followed(db: Session, channel_id: str) -> bool:
    """Whether anyone subscribes to the channel."""
    return (
        db.execute(
            select(Subscription.id).where(Subscription.channel_id == channel_id).limit(1)
        ).first()
        is not None
    )


# --- a user's subscriptions, blocks and explorations ----------------------------------------


def subscriptions(db: Session, user: User, lang: str) -> list[tuple[Subscription, Channel]]:
    """The user's subscriptions with their channels, highest priority first, then by title in the
    reader's alphabet."""
    return list(
        db.execute(
            select(Subscription, Channel)
            .join(Channel, Channel.id == Subscription.channel_id)
            .where(owned_by(Subscription.user_id, user))
            .order_by(Subscription.priority.desc(), collated(Channel.title, lang))
        ).tuples()
    )


def subscription(db: Session, user: User, channel_id: str) -> Subscription | None:
    return db.execute(
        select(Subscription).where(
            owned_by(Subscription.user_id, user), Subscription.channel_id == channel_id
        )
    ).scalar_one_or_none()


def subscribed_channels(user: User):
    """The channel ids the user subscribes to, as a subquery."""
    return select(Subscription.channel_id).where(owned_by(Subscription.user_id, user))


def exploration(db: Session, user: User, channel_id: str) -> ExploredChannel | None:
    return db.execute(
        select(ExploredChannel).where(
            owned_by(ExploredChannel.user_id, user), ExploredChannel.channel_id == channel_id
        )
    ).scalar_one_or_none()


def is_blocked(db: Session, user: User, channel_id: str) -> bool:
    return (
        db.execute(
            select(BlockedChannel.id).where(
                owned_by(BlockedChannel.user_id, user), BlockedChannel.channel_id == channel_id
            )
        ).first()
        is not None
    )


def blocked_ids(db: Session, user: User) -> set[str]:
    return set(
        db.scalars(select(BlockedChannel.channel_id).where(owned_by(BlockedChannel.user_id, user)))
    )


def blocked_channels(db: Session, user: User, lang: str) -> list[Channel]:
    return list(
        db.execute(
            select(Channel)
            .join(BlockedChannel, BlockedChannel.channel_id == Channel.id)
            .where(owned_by(BlockedChannel.user_id, user))
            .order_by(collated(Channel.title, lang))
        ).scalars()
    )


# --- sync progress ---------------------------------------------------------------------------


def catalog_totals(db: Session, user: User) -> dict:
    """The instance's catalog size and this user's subscription count (the sync status card)."""
    return {
        "subscriptions": db.scalar(
            select(func.count())
            .select_from(Subscription)
            .where(owned_by(Subscription.user_id, user))
        ),
        "channels_total": db.scalar(select(func.count()).select_from(Channel)),
        "channels_backfilling": db.scalar(
            select(func.count()).select_from(Channel).where(Channel.backfill_done.is_(False))
        ),
        "videos_total": db.scalar(select(func.count()).select_from(Video)),
        "pending_enrich": db.scalar(
            select(func.count()).select_from(Video).where(Video.enriched_at.is_(None))
        ),
    }


def my_progress(db: Session, user: User) -> tuple[int, int, int, int]:
    """Over the channels this user subscribes to: how many, how many have details, how many their
    recent uploads, how many their whole history."""
    total, details_synced, recent_synced, all_deep_done = db.execute(
        select(
            func.count(Channel.id),
            func.count(Channel.details_synced_at),
            func.count(Channel.recent_synced_at),
            func.sum(case((Channel.backfill_done.is_(True), 1), else_=0)),
        )
        .select_from(Channel)
        .join(
            Subscription,
            and_(Subscription.channel_id == Channel.id, owned_by(Subscription.user_id, user)),
        )
    ).one()
    return total or 0, details_synced or 0, recent_synced or 0, int(all_deep_done or 0)


def my_deep_requested(db: Session, user: User) -> list[Channel]:
    """The channels this user asked full history for."""
    return list(
        db.execute(
            select(Channel).join(
                Subscription,
                and_(
                    Subscription.channel_id == Channel.id,
                    owned_by(Subscription.user_id, user),
                    Subscription.deep_requested.is_(True),
                ),
            )
        ).scalars()
    )


def my_video_count(db: Session, user: User) -> int:
    """The videos of the channels this user subscribes to and has not hidden."""
    return (
        db.scalar(
            select(func.count(Video.id)).join(
                Subscription,
                and_(
                    Subscription.channel_id == Video.channel_id,
                    owned_by(Subscription.user_id, user),
                    Subscription.hidden.is_(False),
                ),
            )
        )
        or 0
    )


def video_total(db: Session) -> int:
    return db.scalar(select(func.count(Video.id))) or 0


def deep_backlog(db: Session) -> list[Channel]:
    """Every channel someone asked full history for that does not have it yet."""
    deep_requested = (
        select(Subscription.id)
        .where(Subscription.channel_id == Channel.id, Subscription.deep_requested.is_(True))
        .exists()
    )
    return list(
        db.execute(
            select(Channel).where(Channel.backfill_done.is_(False), deep_requested)
        ).scalars()
    )


def work_queue(db: Session) -> dict:
    """What the catalog's jobs still have queued, instance-wide (the scheduler card)."""
    return {
        # The same selection run_recent_backfill works through, or the card counts channels the
        # job will never touch and never reaches zero.
        "channels_recent_pending": db.scalar(
            select(func.count())
            .select_from(Channel)
            .where(Channel.recent_synced_at.is_(None), followed_channel())
        )
        or 0,
        "videos_pending_enrich": db.scalar(
            select(func.count()).select_from(Video).where(Video.enriched_at.is_(None))
        )
        or 0,
        "videos_pending_shorts": db.scalar(
            select(func.count())
            .select_from(Video)
            .where(Video.shorts_probed.is_(False), Video.enriched_at.is_not(None))
        )
        or 0,
        "videos_live_refresh": db.scalar(
            select(func.count())
            .select_from(Video)
            .where(
                or_(
                    Video.live_status.in_(("live", "upcoming")),
                    and_(Video.live_status == "was_live", Video.duration_seconds.is_(None)),
                )
            )
        )
        or 0,
    }