feat(reader): the channels a Takeout history shows watched but not followed
The import asks YouTube how long the watched videos of the candidate channels are (five or more watched, none the user follows, blocked, put out or owns), keeps the length on the Takeout row and names the channels that make the list: five videos of five minutes or more. The shorts job asks what an import could not. A channel's counted videos come in as an exploration's when its list is opened, ready to play; a channel can be put out of the list and back.
This commit is contained in:
@@ -0,0 +1,49 @@
|
|||||||
|
"""A watched Takeout video's length, and the channels a user put out of their list
|
||||||
|
|
||||||
|
Revision ID: 0115_takeout_channels
|
||||||
|
Revises: 0114_video_unmarks
|
||||||
|
Create Date: 2026-10-24
|
||||||
|
|
||||||
|
The "channels you watch but do not follow" list counts a user's watched videos over five minutes
|
||||||
|
by channel. The length is asked of YouTube for that list and kept on the Takeout row; a channel the
|
||||||
|
user does not want on the list is kept apart.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0115_takeout_channels"
|
||||||
|
down_revision: str | None = "0114_video_unmarks"
|
||||||
|
branch_labels: str | None = None
|
||||||
|
depends_on: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"takeout_items", sa.Column("looked_up_at", sa.DateTime(timezone=True), nullable=True)
|
||||||
|
)
|
||||||
|
op.add_column("takeout_items", sa.Column("duration_seconds", sa.Integer(), nullable=True))
|
||||||
|
op.create_table(
|
||||||
|
"takeout_channel_dismissals",
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column(
|
||||||
|
"user_id",
|
||||||
|
sa.Integer(),
|
||||||
|
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("channel_id", sa.String(32), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"created_at",
|
||||||
|
sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.func.now(),
|
||||||
|
nullable=False,
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint("user_id", "channel_id", name="uq_takeout_channel_dismissal"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("takeout_channel_dismissals")
|
||||||
|
op.drop_column("takeout_items", "duration_seconds")
|
||||||
|
op.drop_column("takeout_items", "looked_up_at")
|
||||||
@@ -254,8 +254,9 @@ class TakeoutItem(Base, TimestampMixin):
|
|||||||
"""One video from a user's Google Takeout export: watched (with when) or in their YouTube Watch
|
"""One video from a user's Google Takeout export: watched (with when) or in their YouTube Watch
|
||||||
later list (with when it was added). The rows are both the import job's input and its memory: a
|
later list (with when it was added). The rows are both the import job's input and its memory: a
|
||||||
video the catalog does not know yet waits here, with no outcome, until it enters the catalog.
|
video the catalog does not know yet waits here, with no outcome, until it enters the catalog.
|
||||||
The id is not a foreign key for that reason. The channel is kept for a later "channels you
|
The id is not a foreign key for that reason. A watched row's channel and length feed the
|
||||||
watch but do not follow" list."""
|
"channels you watch but do not follow" list: the length is asked of YouTube for that list alone
|
||||||
|
and kept here, so its videos need not enter the shared catalog to be counted."""
|
||||||
|
|
||||||
__tablename__ = "takeout_items"
|
__tablename__ = "takeout_items"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
@@ -277,3 +278,21 @@ class TakeoutItem(Base, TimestampMixin):
|
|||||||
# Watched: the last time in the export; Watch later: when it was added.
|
# Watched: the last time in the export; Watch later: when it was added.
|
||||||
at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||||
outcome: Mapped[str | None] = mapped_column(String(12))
|
outcome: Mapped[str | None] = mapped_column(String(12))
|
||||||
|
# When YouTube was asked for the video's length (NULL = not yet), and the length it gave; a
|
||||||
|
# video it no longer has keeps a NULL length.
|
||||||
|
looked_up_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||||
|
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||||
|
|
||||||
|
|
||||||
|
class TakeoutChannelDismissal(Base, TimestampMixin):
|
||||||
|
"""A channel the user put out of their "watched but not followed" list. The id is not a foreign
|
||||||
|
key: the channel may never have entered the catalog, or may leave it, and the choice stays."""
|
||||||
|
|
||||||
|
__tablename__ = "takeout_channel_dismissals"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("user_id", "channel_id", name="uq_takeout_channel_dismissal"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(primary_key=True)
|
||||||
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||||
|
channel_id: Mapped[str] = mapped_column(String(32))
|
||||||
|
|||||||
@@ -16,13 +16,14 @@ from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
|
|||||||
from app.collation import collated
|
from app.collation import collated
|
||||||
from app.feedquery import feed_columns, serialize
|
from app.feedquery import feed_columns, serialize
|
||||||
from app.models import User
|
from app.models import User
|
||||||
from app.catalog.models import Channel, Subscription, Video
|
from app.catalog.models import BlockedChannel, Channel, Subscription, Video
|
||||||
from app.reader.models import (
|
from app.reader.models import (
|
||||||
ChannelTag,
|
ChannelTag,
|
||||||
Playlist,
|
Playlist,
|
||||||
PlaylistItem,
|
PlaylistItem,
|
||||||
SavedView,
|
SavedView,
|
||||||
Tag,
|
Tag,
|
||||||
|
TakeoutChannelDismissal,
|
||||||
TakeoutItem,
|
TakeoutItem,
|
||||||
TakeoutKind,
|
TakeoutKind,
|
||||||
VideoState,
|
VideoState,
|
||||||
@@ -533,3 +534,164 @@ def takeout_users_watch_later_waiting(db: Session) -> list[int]:
|
|||||||
.distinct()
|
.distinct()
|
||||||
).scalars()
|
).scalars()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --- channels watched but not followed ---------------------------------------------------------
|
||||||
|
|
||||||
|
# A channel joins the list with this many watched videos of at least this length. A Short is three
|
||||||
|
# minutes at most, so the length alone keeps them out, and a video under five minutes says too
|
||||||
|
# little of what a channel is about to count towards it.
|
||||||
|
WATCHED_CHANNEL_MIN_VIDEOS = 5
|
||||||
|
WATCHED_CHANNEL_MIN_SECONDS = 300
|
||||||
|
|
||||||
|
|
||||||
|
def _unfollowed(user: User) -> ColumnElement[bool]:
|
||||||
|
"""A watched row whose channel the user neither follows, blocked, put out of the list, nor is."""
|
||||||
|
subscribed = select(Subscription.channel_id).where(owned_by(Subscription.user_id, user))
|
||||||
|
blocked = select(BlockedChannel.channel_id).where(owned_by(BlockedChannel.user_id, user))
|
||||||
|
dismissed = select(TakeoutChannelDismissal.channel_id).where(
|
||||||
|
owned_by(TakeoutChannelDismissal.user_id, user)
|
||||||
|
)
|
||||||
|
clauses = [
|
||||||
|
owned_by(TakeoutItem.user_id, user),
|
||||||
|
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||||
|
TakeoutItem.channel_id.is_not(None),
|
||||||
|
TakeoutItem.channel_id.not_in(subscribed),
|
||||||
|
TakeoutItem.channel_id.not_in(blocked),
|
||||||
|
TakeoutItem.channel_id.not_in(dismissed),
|
||||||
|
]
|
||||||
|
if user.yt_channel_id:
|
||||||
|
clauses.append(TakeoutItem.channel_id != user.yt_channel_id)
|
||||||
|
return and_(*clauses)
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_channels(user: User):
|
||||||
|
"""The unfollowed channels with enough watched videos to make the list once their lengths are
|
||||||
|
known: (channel id, how many watched)."""
|
||||||
|
return (
|
||||||
|
select(TakeoutItem.channel_id, func.count().label("watched"))
|
||||||
|
.where(_unfollowed(user))
|
||||||
|
.group_by(TakeoutItem.channel_id)
|
||||||
|
.having(func.count() >= WATCHED_CHANNEL_MIN_VIDEOS)
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def takeout_lengths_to_ask(db: Session, user: User) -> list[str]:
|
||||||
|
"""The watched videos whose length the list still needs: those of the candidate channels not
|
||||||
|
asked about yet, the most watched channels first, so a lookup cut short fills the top."""
|
||||||
|
cand = _candidate_channels(user)
|
||||||
|
return list(
|
||||||
|
db.execute(
|
||||||
|
select(TakeoutItem.video_id)
|
||||||
|
.join(cand, cand.c.channel_id == TakeoutItem.channel_id)
|
||||||
|
.where(_unfollowed(user), TakeoutItem.looked_up_at.is_(None))
|
||||||
|
.order_by(cand.c.watched.desc(), TakeoutItem.channel_id, TakeoutItem.at.desc())
|
||||||
|
).scalars()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def watched_channels_pending(db: Session, user: User) -> int:
|
||||||
|
"""How many candidate channels still have a watched video whose length is not known: the
|
||||||
|
channels that may yet join the list."""
|
||||||
|
cand = _candidate_channels(user)
|
||||||
|
return (
|
||||||
|
db.scalar(
|
||||||
|
select(func.count(func.distinct(TakeoutItem.channel_id)))
|
||||||
|
.join(cand, cand.c.channel_id == TakeoutItem.channel_id)
|
||||||
|
.where(_unfollowed(user), TakeoutItem.looked_up_at.is_(None))
|
||||||
|
)
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def watched_channels_to_describe(db: Session, user: User) -> list[str]:
|
||||||
|
"""The channels on the list (by their counted videos) that the catalog lacks or has no details
|
||||||
|
of yet: what the list cannot name."""
|
||||||
|
counted = (
|
||||||
|
select(TakeoutItem.channel_id)
|
||||||
|
.where(_unfollowed(user), TakeoutItem.duration_seconds >= WATCHED_CHANNEL_MIN_SECONDS)
|
||||||
|
.group_by(TakeoutItem.channel_id)
|
||||||
|
.having(func.count() >= WATCHED_CHANNEL_MIN_VIDEOS)
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
return list(
|
||||||
|
db.execute(
|
||||||
|
select(counted.c.channel_id)
|
||||||
|
.outerjoin(Channel, Channel.id == counted.c.channel_id)
|
||||||
|
.where(Channel.details_synced_at.is_(None))
|
||||||
|
).scalars()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def watched_channels(db: Session, user: User) -> list[tuple[Channel, int, datetime]]:
|
||||||
|
"""The channels the user watched often but does not follow: (channel, how many of its videos
|
||||||
|
over five minutes they watched, when last). Most watched first; on a tie the one watched more
|
||||||
|
lately, which is the likelier interest."""
|
||||||
|
n = func.count(TakeoutItem.id)
|
||||||
|
last = func.max(TakeoutItem.at)
|
||||||
|
return list(
|
||||||
|
db.execute(
|
||||||
|
select(Channel, n, last)
|
||||||
|
.join(Channel, Channel.id == TakeoutItem.channel_id)
|
||||||
|
.where(_unfollowed(user), TakeoutItem.duration_seconds >= WATCHED_CHANNEL_MIN_SECONDS)
|
||||||
|
.group_by(Channel.id)
|
||||||
|
.having(n >= WATCHED_CHANNEL_MIN_VIDEOS)
|
||||||
|
.order_by(n.desc(), last.desc(), Channel.id)
|
||||||
|
).tuples()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def has_watch_history(db: Session, user: User) -> bool:
|
||||||
|
"""Whether the user has uploaded a Takeout watch history at all."""
|
||||||
|
return bool(
|
||||||
|
db.scalar(
|
||||||
|
select(
|
||||||
|
select(TakeoutItem.id)
|
||||||
|
.where(owned_by(TakeoutItem.user_id, user), TakeoutItem.kind == TakeoutKind.WATCHED)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def watched_channel_videos(
|
||||||
|
db: Session, user: User, channel_id: str
|
||||||
|
) -> tuple[list[tuple[str, datetime]], int]:
|
||||||
|
"""The user's watched videos of one channel that the list counts, last watched first: (video
|
||||||
|
id, when). And how many YouTube no longer has."""
|
||||||
|
rows = db.execute(
|
||||||
|
select(TakeoutItem.video_id, TakeoutItem.at, TakeoutItem.duration_seconds)
|
||||||
|
.where(
|
||||||
|
owned_by(TakeoutItem.user_id, user),
|
||||||
|
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||||
|
TakeoutItem.channel_id == channel_id,
|
||||||
|
TakeoutItem.looked_up_at.is_not(None),
|
||||||
|
)
|
||||||
|
.order_by(TakeoutItem.at.desc(), TakeoutItem.video_id)
|
||||||
|
).tuples()
|
||||||
|
counted: list[tuple[str, datetime]] = []
|
||||||
|
gone = 0
|
||||||
|
for vid, at, seconds in rows:
|
||||||
|
if seconds is None:
|
||||||
|
gone += 1
|
||||||
|
elif seconds >= WATCHED_CHANNEL_MIN_SECONDS:
|
||||||
|
counted.append((vid, at))
|
||||||
|
return counted, gone
|
||||||
|
|
||||||
|
|
||||||
|
def takeout_users_with_lengths_to_ask(db: Session) -> list[int]:
|
||||||
|
"""Every user with a channel of enough watched videos not all asked about yet. It does not ask
|
||||||
|
whether they follow the channel: that is each user's own question, which
|
||||||
|
`takeout_lengths_to_ask` answers (and an empty answer costs a run nothing)."""
|
||||||
|
per_channel = (
|
||||||
|
select(TakeoutItem.user_id)
|
||||||
|
.where(TakeoutItem.kind == TakeoutKind.WATCHED, TakeoutItem.channel_id.is_not(None))
|
||||||
|
.group_by(TakeoutItem.user_id, TakeoutItem.channel_id)
|
||||||
|
.having(
|
||||||
|
func.count() >= WATCHED_CHANNEL_MIN_VIDEOS,
|
||||||
|
func.count().filter(TakeoutItem.looked_up_at.is_(None)) > 0,
|
||||||
|
)
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
return list(db.execute(select(per_channel.c.user_id).distinct()).scalars())
|
||||||
|
|||||||
@@ -9,7 +9,9 @@ unique key, a row lock taken before a presence check. The comfort shelf's writes
|
|||||||
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
from sqlalchemy import exists, func, select, text, update
|
from typing import cast
|
||||||
|
|
||||||
|
from sqlalchemy import Table, bindparam, delete, exists, func, select, text, update
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
@@ -21,6 +23,7 @@ from app.reader.models import (
|
|||||||
PlaylistItem,
|
PlaylistItem,
|
||||||
SavedView,
|
SavedView,
|
||||||
Tag,
|
Tag,
|
||||||
|
TakeoutChannelDismissal,
|
||||||
TakeoutItem,
|
TakeoutItem,
|
||||||
TakeoutKind,
|
TakeoutKind,
|
||||||
TakeoutOutcome,
|
TakeoutOutcome,
|
||||||
@@ -662,3 +665,51 @@ def apply_takeout_watch_later(
|
|||||||
"watch_later_present": len(done[TakeoutOutcome.PRESENT]),
|
"watch_later_present": len(done[TakeoutOutcome.PRESENT]),
|
||||||
"watch_later_missing": len(done[TakeoutOutcome.MISSING]),
|
"watch_later_missing": len(done[TakeoutOutcome.MISSING]),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# --- channels watched but not followed ---------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def record_takeout_lengths(
|
||||||
|
db: Session, user: User, asked: list[str], lengths: dict[str, int | None]
|
||||||
|
) -> None:
|
||||||
|
"""Keep what YouTube said of the user's watched videos `asked` about: each one's length, or none
|
||||||
|
for a video it no longer has (`lengths` leaves it out). Every asked row is marked asked, so a
|
||||||
|
video YouTube lost is not asked about again."""
|
||||||
|
if not asked:
|
||||||
|
return
|
||||||
|
lock_takeout(db, user) # these rows are an upload's to update too
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
table = cast(Table, TakeoutItem.__table__)
|
||||||
|
db.execute(
|
||||||
|
table.update()
|
||||||
|
.where(
|
||||||
|
owned_by(TakeoutItem.user_id, user),
|
||||||
|
table.c.kind == TakeoutKind.WATCHED,
|
||||||
|
table.c.video_id == bindparam("vid"),
|
||||||
|
)
|
||||||
|
.values(looked_up_at=now, duration_seconds=bindparam("seconds")),
|
||||||
|
[{"vid": vid, "seconds": lengths.get(vid)} for vid in asked],
|
||||||
|
)
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def dismiss_watched_channel(db: Session, user: User, channel_id: str) -> None:
|
||||||
|
"""Put a channel out of the user's "watched but not followed" list (again is a no-op)."""
|
||||||
|
db.execute(
|
||||||
|
pg_insert(TakeoutChannelDismissal)
|
||||||
|
.values(user_id=user.id, channel_id=channel_id)
|
||||||
|
.on_conflict_do_nothing(constraint="uq_takeout_channel_dismissal")
|
||||||
|
)
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def restore_watched_channel(db: Session, user: User, channel_id: str) -> None:
|
||||||
|
"""Let a channel the user put out of the list back on it."""
|
||||||
|
db.execute(
|
||||||
|
delete(TakeoutChannelDismissal).where(
|
||||||
|
owned_by(TakeoutChannelDismissal.user_id, user),
|
||||||
|
TakeoutChannelDismissal.channel_id == channel_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
db.flush()
|
||||||
|
|||||||
@@ -4,7 +4,9 @@ only way in.
|
|||||||
|
|
||||||
Two files of the export are read, each uploaded on its own: `history/watch-history.json` and
|
Two files of the export are read, each uploaded on its own: `history/watch-history.json` and
|
||||||
`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies
|
`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies
|
||||||
what it can; a video the catalog does not know yet waits for it.
|
what it can; a video the catalog does not know yet waits for it. The watched videos' channels also
|
||||||
|
make the list of channels the user watches but does not follow (`queries.watched_channels`): the
|
||||||
|
job asks YouTube how long those videos are, and what the channels are, for it.
|
||||||
|
|
||||||
The format, measured on a real export (2026-10): the history is a JSON list, one entry per view,
|
The format, measured on a real export (2026-10): the history is a JSON list, one entry per view,
|
||||||
newest first. A view of a video has a `titleUrl` with `watch?v=<id>` (on www.youtube.com or
|
newest first. A view of a video has a `titleUrl` with `watch?v=<id>` (on www.youtube.com or
|
||||||
@@ -37,8 +39,9 @@ from app.ratelimit import RateLimiter
|
|||||||
from app.reader import queries, service
|
from app.reader import queries, service
|
||||||
from app.reader.models import TakeoutKind
|
from app.reader.models import TakeoutKind
|
||||||
from app.sync.runner import get_service_user
|
from app.sync.runner import get_service_user
|
||||||
from app.sync.videos import ensure_videos
|
from app.sync.subscriptions import describe_channels
|
||||||
from app.youtube.client import YouTubeClient
|
from app.sync.videos import ensure_videos, parse_iso8601_duration
|
||||||
|
from app.youtube.client import YouTubeClient, YouTubeError
|
||||||
|
|
||||||
log = logs.get_logger(__name__)
|
log = logs.get_logger(__name__)
|
||||||
|
|
||||||
@@ -351,10 +354,79 @@ def _put_in_watch_later(db: Session, user: User) -> dict:
|
|||||||
return service.apply_takeout_watch_later(db, user, waiting, asked)
|
return service.apply_takeout_watch_later(db, user, waiting, asked)
|
||||||
|
|
||||||
|
|
||||||
|
def _ask_lengths(db: Session, user: User) -> dict:
|
||||||
|
"""Ask YouTube how long the user's watched videos are that the "watched but not followed" list
|
||||||
|
still needs, and keep the answer — on the same terms and the same daily share as the Watch
|
||||||
|
later lookups. What is not asked yet (no YouTube-connected account, the reserve reached, the
|
||||||
|
share spent) waits for a later run, and the list says it is not complete. A YouTube failure is
|
||||||
|
logged and left for a later run too: it is no reason to fail the import it follows. The caller
|
||||||
|
commits."""
|
||||||
|
ask = queries.takeout_lengths_to_ask(db, user)
|
||||||
|
fetcher = get_service_user(db) if ask else None
|
||||||
|
if fetcher is None or quota.backfill_reserve_reached():
|
||||||
|
return {}
|
||||||
|
ask = ask[: int(_lookups.room(str(user.id)))]
|
||||||
|
if not ask:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP):
|
||||||
|
with YouTubeClient(db, fetcher) as yt:
|
||||||
|
items = yt.get_videos(ask)
|
||||||
|
except YouTubeError as exc:
|
||||||
|
log.warning(
|
||||||
|
"Takeout: asking the watched videos' lengths failed: %s",
|
||||||
|
exc,
|
||||||
|
extra=logs.event("takeout.lengths.failed", user_id=user.id, reason=exc.reason),
|
||||||
|
)
|
||||||
|
return {}
|
||||||
|
_lookups.allow(str(user.id), cost=len(ask))
|
||||||
|
lengths = {
|
||||||
|
it["id"]: parse_iso8601_duration(it.get("contentDetails", {}).get("duration"))
|
||||||
|
for it in items
|
||||||
|
if it.get("id")
|
||||||
|
}
|
||||||
|
service.record_takeout_lengths(db, user, ask, lengths)
|
||||||
|
return {"lengths_asked": len(ask)}
|
||||||
|
|
||||||
|
|
||||||
|
def _describe_channels(db: Session, user: User) -> dict:
|
||||||
|
"""Bring the channels the user's list holds into the catalog with their details (name, picture,
|
||||||
|
description), the ones it lacks: without them the list cannot show a channel. A YouTube failure
|
||||||
|
is logged and left for a later run. The caller commits."""
|
||||||
|
need = queries.watched_channels_to_describe(db, user)
|
||||||
|
fetcher = get_service_user(db) if need else None
|
||||||
|
if fetcher is None or quota.backfill_reserve_reached():
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
|
||||||
|
with YouTubeClient(db, fetcher) as yt:
|
||||||
|
describe_channels(db, yt, need)
|
||||||
|
except YouTubeError as exc:
|
||||||
|
db.rollback()
|
||||||
|
log.warning(
|
||||||
|
"Takeout: describing the watched channels failed: %s",
|
||||||
|
exc,
|
||||||
|
extra=logs.event("takeout.channels.failed", user_id=user.id, reason=exc.reason),
|
||||||
|
)
|
||||||
|
return {}
|
||||||
|
return {"channels_described": len(need)}
|
||||||
|
|
||||||
|
|
||||||
|
def _channel_list(db: Session, user: User) -> dict:
|
||||||
|
"""The "watched but not followed" list's part of a run: the lengths, landed, then the channels.
|
||||||
|
Commits each."""
|
||||||
|
result = _ask_lengths(db, user)
|
||||||
|
db.commit()
|
||||||
|
result |= _describe_channels(db, user)
|
||||||
|
db.commit()
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def run_import(user_id: int) -> dict:
|
def run_import(user_id: int) -> dict:
|
||||||
"""The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
|
"""The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
|
||||||
The watched part commits before the Watch later part asks YouTube for the videos the catalog
|
The watched part commits before the Watch later part asks YouTube for the videos the catalog
|
||||||
lacks, so a YouTube failure there loses nothing already marked. Returns the counts."""
|
lacks, so a YouTube failure there loses nothing already marked; the "watched but not followed"
|
||||||
|
list's lookups come last. Returns the counts."""
|
||||||
with _one_import(user_id), SessionLocal() as db:
|
with _one_import(user_id), SessionLocal() as db:
|
||||||
user = db.get(User, user_id)
|
user = db.get(User, user_id)
|
||||||
if user is None:
|
if user is None:
|
||||||
@@ -363,6 +435,7 @@ def run_import(user_id: int) -> dict:
|
|||||||
db.commit()
|
db.commit()
|
||||||
result |= _put_in_watch_later(db, user)
|
result |= _put_in_watch_later(db, user)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
result |= _channel_list(db, user)
|
||||||
outcomes = queries.takeout_outcomes(db, user)
|
outcomes = queries.takeout_outcomes(db, user)
|
||||||
result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
|
result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
|
||||||
if outcomes.get((TakeoutKind.WATCH_LATER, None)):
|
if outcomes.get((TakeoutKind.WATCH_LATER, None)):
|
||||||
@@ -373,13 +446,15 @@ def run_import(user_id: int) -> dict:
|
|||||||
def apply_waiting(db: Session) -> dict:
|
def apply_waiting(db: Session) -> dict:
|
||||||
"""Settle what every user's import left waiting and can be settled now, one commit each: a
|
"""Settle what every user's import left waiting and can be settled now, one commit each: a
|
||||||
watched video the catalog has come to know and the Shorts probe has cleared (by its job or a
|
watched video the catalog has come to know and the Shorts probe has cleared (by its job or a
|
||||||
search's), and the Watch later videos a run could not put in yet. Run after each run of the
|
search's), the Watch later videos a run could not put in yet, and what the "watched but not
|
||||||
|
followed" list still needs asked of YouTube. Run after each run of the
|
||||||
`shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when
|
`shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when
|
||||||
nothing was waiting that could move."""
|
nothing was waiting that could move."""
|
||||||
marked = added = failed = 0
|
marked = added = asked = failed = 0
|
||||||
ready = set(queries.takeout_users_ready(db))
|
ready = set(queries.takeout_users_ready(db))
|
||||||
later = set(queries.takeout_users_watch_later_waiting(db))
|
later = set(queries.takeout_users_watch_later_waiting(db))
|
||||||
for user_id in sorted(ready | later):
|
lengths = set(queries.takeout_users_with_lengths_to_ask(db))
|
||||||
|
for user_id in sorted(ready | later | lengths):
|
||||||
user = db.get(User, user_id)
|
user = db.get(User, user_id)
|
||||||
if user is None:
|
if user is None:
|
||||||
continue
|
continue
|
||||||
@@ -393,6 +468,8 @@ def apply_waiting(db: Session) -> dict:
|
|||||||
if user_id in later:
|
if user_id in later:
|
||||||
added += _put_in_watch_later(db, user).get("watch_later_added", 0)
|
added += _put_in_watch_later(db, user).get("watch_later_added", 0)
|
||||||
db.commit()
|
db.commit()
|
||||||
|
if user_id in lengths:
|
||||||
|
asked += _channel_list(db, user).get("lengths_asked", 0)
|
||||||
except Exception:
|
except Exception:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
failed += 1
|
failed += 1
|
||||||
@@ -406,6 +483,8 @@ def apply_waiting(db: Session) -> dict:
|
|||||||
out["takeout_watched"] = marked
|
out["takeout_watched"] = marked
|
||||||
if added:
|
if added:
|
||||||
out["takeout_watch_later_added"] = added
|
out["takeout_watch_later_added"] = added
|
||||||
|
if asked:
|
||||||
|
out["takeout_lengths_asked"] = asked
|
||||||
if failed:
|
if failed:
|
||||||
out["takeout_failed"] = failed
|
out["takeout_failed"] = failed
|
||||||
return out
|
return out
|
||||||
|
|||||||
@@ -3,12 +3,13 @@ from collections.abc import AsyncIterator
|
|||||||
from datetime import date
|
from datetime import date
|
||||||
from functools import partial
|
from functools import partial
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, Query, Request
|
from fastapi import APIRouter, Depends, Path, Query, Request
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from sqlalchemy.orm.exc import StaleDataError
|
from sqlalchemy.orm.exc import StaleDataError
|
||||||
|
|
||||||
from app import jobruns, quota, scheduler
|
from app import jobruns, quota, scheduler
|
||||||
|
from app.catalog import queries as catalog_queries, service as catalog_service
|
||||||
from app.auth import current_user
|
from app.auth import current_user
|
||||||
from app.db import get_db, release_read_connection
|
from app.db import get_db, release_read_connection
|
||||||
from app.mailtext import lang_for_user
|
from app.mailtext import lang_for_user
|
||||||
@@ -24,7 +25,7 @@ from app.routes._common import (
|
|||||||
not_found,
|
not_found,
|
||||||
youtube_refusal,
|
youtube_refusal,
|
||||||
)
|
)
|
||||||
from app.sync.videos import parse_iso8601_duration
|
from app.sync.videos import ensure_videos, parse_iso8601_duration
|
||||||
from app.utils import iso
|
from app.utils import iso
|
||||||
from app.youtube.client import YouTubeClient, YouTubeError
|
from app.youtube.client import YouTubeClient, YouTubeError
|
||||||
from app.refusal import refused
|
from app.refusal import refused
|
||||||
@@ -417,3 +418,95 @@ def takeout_status(user: User = Depends(current_user), db: Session = Depends(get
|
|||||||
},
|
},
|
||||||
**queries.takeout_summary(db, user),
|
**queries.takeout_summary(db, user),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# --- channels watched but not followed -----------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/takeout/channels")
|
||||||
|
def watched_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||||
|
"""The channels the user's Takeout history shows them watching often but not following: each
|
||||||
|
with how many of its videos over five minutes they watched, and when last. `pending` is how many
|
||||||
|
more channels may join once the import has asked YouTube for their videos' lengths, so a list
|
||||||
|
still being made does not read as the whole of it; `imported` tells "no history uploaded" from
|
||||||
|
"nothing to suggest"."""
|
||||||
|
return {
|
||||||
|
"imported": queries.has_watch_history(db, user),
|
||||||
|
"pending": queries.watched_channels_pending(db, user),
|
||||||
|
"channels": [
|
||||||
|
{
|
||||||
|
"id": ch.id,
|
||||||
|
"title": ch.title,
|
||||||
|
"handle": ch.handle,
|
||||||
|
"thumbnail_url": ch.thumbnail_url,
|
||||||
|
"subscriber_count": ch.subscriber_count,
|
||||||
|
"video_count": ch.video_count,
|
||||||
|
"description": ch.description,
|
||||||
|
"watched": int(n),
|
||||||
|
"last_watched_at": iso(last),
|
||||||
|
}
|
||||||
|
for ch, n, last in queries.watched_channels(db, user)
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/takeout/channels/{channel_id}/videos")
|
||||||
|
def watched_channel_videos(
|
||||||
|
channel_id: str, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||||
|
) -> dict:
|
||||||
|
"""The videos of a channel that the user's list counts, ready to play, last watched first. The
|
||||||
|
channel is explored for the user (as its page does), and the videos the catalog lacks come in as
|
||||||
|
the exploration's: the player needs them in the catalog, the shared Library does not get them,
|
||||||
|
and the cleanup reclaims them once nobody keeps them. `gone`: how many YouTube no longer has."""
|
||||||
|
channel = catalog_queries.channel(db, channel_id)
|
||||||
|
if channel is None:
|
||||||
|
raise not_found("channel")
|
||||||
|
if catalog_queries.is_blocked(db, user, channel_id):
|
||||||
|
raise refused(403, "channel_blocked")
|
||||||
|
counted, gone = queries.watched_channel_videos(db, user, channel_id)
|
||||||
|
ids = [vid for vid, _ in counted]
|
||||||
|
known = queries.known_video_ids(db, ids)
|
||||||
|
missing = [vid for vid in ids if vid not in known]
|
||||||
|
if missing and quota.backfill_reserve_reached():
|
||||||
|
raise refused(429, "quota_reserve")
|
||||||
|
catalog_service.explore(db, user, channel)
|
||||||
|
db.commit()
|
||||||
|
if missing:
|
||||||
|
try:
|
||||||
|
with (
|
||||||
|
quota.attribute(user.id, quota.QuotaAction.CHANNELS_EXPLORE),
|
||||||
|
YouTubeClient(db, user) as yt,
|
||||||
|
):
|
||||||
|
ensure_videos(db, yt, missing, explore=True)
|
||||||
|
except YouTubeError as exc:
|
||||||
|
raise youtube_refusal(exc) from exc
|
||||||
|
items = queries.feed_rows_by_ids(db, user, ids)
|
||||||
|
watched_at = dict(counted)
|
||||||
|
for item in items:
|
||||||
|
item["takeout_watched_at"] = iso(watched_at[item["id"]])
|
||||||
|
# A video YouTube still had when its length was asked may have gone since.
|
||||||
|
return {"ok": True, "items": items, "gone": gone + len(ids) - len(items)}
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/takeout/channels/{channel_id}/dismissal")
|
||||||
|
def dismiss_watched_channel(
|
||||||
|
channel_id: str = Path(max_length=32),
|
||||||
|
user: User = Depends(current_user),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
"""Put a channel out of the user's list (it stays out until restored)."""
|
||||||
|
service.dismiss_watched_channel(db, user, channel_id)
|
||||||
|
db.commit()
|
||||||
|
return {"ok": True, "channel_id": channel_id}
|
||||||
|
|
||||||
|
|
||||||
|
@router.delete("/takeout/channels/{channel_id}/dismissal")
|
||||||
|
def restore_watched_channel(
|
||||||
|
channel_id: str = Path(max_length=32),
|
||||||
|
user: User = Depends(current_user),
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
"""Let a channel the user put out of the list back on it."""
|
||||||
|
service.restore_watched_channel(db, user, channel_id)
|
||||||
|
db.commit()
|
||||||
|
return {"ok": True, "channel_id": channel_id}
|
||||||
|
|||||||
@@ -109,6 +109,21 @@ def apply_channel_details(db: Session, items: list[dict], asked: Iterable[str])
|
|||||||
channel.details_synced_at = now
|
channel.details_synced_at = now
|
||||||
|
|
||||||
|
|
||||||
|
def describe_channels(db: Session, yt: YouTubeClient, channel_ids: list[str]) -> None:
|
||||||
|
"""Make sure each given channel is in the catalog with its details: a channel it lacks is added
|
||||||
|
as a stub, then all of them are asked for (`channels.list`, 1 unit a 50). Flushes; the caller
|
||||||
|
commits."""
|
||||||
|
if not channel_ids:
|
||||||
|
return
|
||||||
|
have = set(db.execute(select(Channel.id).where(Channel.id.in_(channel_ids))).scalars())
|
||||||
|
for cid in dict.fromkeys(channel_ids):
|
||||||
|
if cid not in have:
|
||||||
|
db.add(Channel(id=cid))
|
||||||
|
db.flush()
|
||||||
|
apply_channel_details(db, yt.get_channels(channel_ids), channel_ids)
|
||||||
|
db.flush()
|
||||||
|
|
||||||
|
|
||||||
def should_prune(fetched_count: int, existing_count: int) -> bool:
|
def should_prune(fetched_count: int, existing_count: int) -> bool:
|
||||||
"""Whether an import may delete the local subscriptions missing from this fetch.
|
"""Whether an import may delete the local subscriptions missing from this fetch.
|
||||||
|
|
||||||
|
|||||||
@@ -420,11 +420,15 @@ def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
|
|||||||
return {"probed": probed, "shorts": shorts}
|
return {"probed": probed, "shorts": shorts}
|
||||||
|
|
||||||
|
|
||||||
def ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
|
def ensure_videos(
|
||||||
|
db: Session, yt: YouTubeClient, video_ids: list[str], *, explore: bool = False
|
||||||
|
) -> None:
|
||||||
"""Make sure every given video id exists in the catalog, fetching + ingesting the
|
"""Make sure every given video id exists in the catalog, fetching + ingesting the
|
||||||
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
|
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
|
||||||
the API won't return are simply left out. The catalog's write for a playlist sync that names
|
the API won't return are simply left out. The catalog's write for a playlist sync that names
|
||||||
videos it has never seen; commits."""
|
videos it has never seen; commits. `explore` adds the new ones as an exploration's
|
||||||
|
(`via_explore`): visible to who explores or follows their channel, and reclaimed by the cleanup
|
||||||
|
once nobody keeps them."""
|
||||||
if not video_ids:
|
if not video_ids:
|
||||||
return
|
return
|
||||||
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
|
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
|
||||||
@@ -458,7 +462,12 @@ def ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
|
|||||||
seen.add(vid)
|
seen.add(vid)
|
||||||
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
|
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
|
||||||
# fallback here too so the constructor never carries an explicit NULL.
|
# fallback here too so the constructor never carries an explicit NULL.
|
||||||
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
|
v = Video(
|
||||||
|
id=vid,
|
||||||
|
channel_id=cid,
|
||||||
|
published_at=parse_dt(sn.get("publishedAt")) or now,
|
||||||
|
via_explore=explore,
|
||||||
|
)
|
||||||
apply_video_details(v, it)
|
apply_video_details(v, it)
|
||||||
v.enriched_at = now
|
v.enriched_at = now
|
||||||
db.add(v)
|
db.add(v)
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ UNIT_MODELS = {
|
|||||||
"SavedView",
|
"SavedView",
|
||||||
"ComfortPin",
|
"ComfortPin",
|
||||||
"TakeoutItem",
|
"TakeoutItem",
|
||||||
|
"TakeoutChannelDismissal",
|
||||||
"VideoUnmark",
|
"VideoUnmark",
|
||||||
"Tag",
|
"Tag",
|
||||||
"ChannelTag",
|
"ChannelTag",
|
||||||
@@ -367,6 +368,8 @@ UNIT_COMMITS = {
|
|||||||
"reader/takeout.py:run_import",
|
"reader/takeout.py:run_import",
|
||||||
# Run inside the Shorts job, on its session: each user's waiting rows land on their own.
|
# Run inside the Shorts job, on its session: each user's waiting rows land on their own.
|
||||||
"reader/takeout.py:apply_waiting",
|
"reader/takeout.py:apply_waiting",
|
||||||
|
# Inside either of those: the lengths YouTube gave land before the channels are asked for.
|
||||||
|
"reader/takeout.py:_channel_list",
|
||||||
},
|
},
|
||||||
# The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page
|
# The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page
|
||||||
# fetched and stored is not lost when the next call fails, and an enrich that fails rolls back
|
# fetched and stored is not lost when the next call fails, and an enrich that fails rolls back
|
||||||
|
|||||||
@@ -0,0 +1,399 @@
|
|||||||
|
"""The channels a user's Takeout history shows them watching but not following: which channels make
|
||||||
|
the list (five videos of five minutes or more, none they follow, block, put out or own), what the
|
||||||
|
import asks YouTube for it and when, and the channel's watched videos made ready to play."""
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
|
from app import quota
|
||||||
|
from app.catalog.models import BlockedChannel, Channel, ExploredChannel, Subscription, Video
|
||||||
|
from app.reader import queries, service, takeout
|
||||||
|
from app.reader.models import TakeoutChannelDismissal, TakeoutItem, TakeoutKind
|
||||||
|
from app.routes import feed as feed_routes
|
||||||
|
from app.sync import videos as sync_videos
|
||||||
|
from app.youtube.client import YouTubeError
|
||||||
|
|
||||||
|
T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
|
||||||
|
LONG = 600
|
||||||
|
SHORT = 120
|
||||||
|
|
||||||
|
|
||||||
|
def _ch(n: int) -> str:
|
||||||
|
return f"UC{n:022d}"
|
||||||
|
|
||||||
|
|
||||||
|
def _vid(channel: int, n: int) -> str:
|
||||||
|
return f"c{channel:02d}v{n:07d}"
|
||||||
|
|
||||||
|
|
||||||
|
def _watched(db, user, channel: int, lengths: list[int | None], *, at=T0, asked=True):
|
||||||
|
"""Stage `len(lengths)` watched videos of a channel, the newest first, each with its length
|
||||||
|
already asked (None: YouTube no longer has it) — or not asked yet."""
|
||||||
|
service.stage_takeout(
|
||||||
|
db,
|
||||||
|
user,
|
||||||
|
TakeoutKind.WATCHED,
|
||||||
|
{_vid(channel, i): (at - timedelta(days=i), _ch(channel)) for i in range(len(lengths))},
|
||||||
|
)
|
||||||
|
db.flush()
|
||||||
|
if asked:
|
||||||
|
ids = [_vid(channel, i) for i in range(len(lengths))]
|
||||||
|
service.record_takeout_lengths(
|
||||||
|
db, user, ids, {v: s for v, s in zip(ids, lengths, strict=True) if s is not None}
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _channel(db, n: int, title: str | None = None):
|
||||||
|
db.add(Channel(id=_ch(n), title=title or f"Channel {n}", details_synced_at=T0))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
|
||||||
|
def _listed(db, user) -> list[tuple[str, int]]:
|
||||||
|
return [(ch.id, n) for ch, n, _ in queries.watched_channels(db, user)]
|
||||||
|
|
||||||
|
|
||||||
|
# --- which channels make the list --------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channel_makes_the_list_with_five_watched_videos_of_five_minutes_or_more(db, make_user):
|
||||||
|
user = make_user("list@example.com")
|
||||||
|
for n in (1, 2, 3):
|
||||||
|
_channel(db, n)
|
||||||
|
_watched(db, user, 1, [LONG] * 5 + [SHORT] * 3) # five long, three short: on it with 5
|
||||||
|
_watched(db, user, 2, [LONG] * 4 + [SHORT] * 6) # ten watched, only four long: off it
|
||||||
|
_watched(db, user, 3, [299, 300, 300, 300, 300, 300]) # five at exactly five minutes: on it
|
||||||
|
# A tie: channel 1's counted videos were watched later (channel 3's newest is under five minutes).
|
||||||
|
assert _listed(db, user) == [(_ch(1), 5), (_ch(3), 5)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_video_youtube_no_longer_has_does_not_count(db, make_user):
|
||||||
|
user = make_user("gone@example.com")
|
||||||
|
_channel(db, 1)
|
||||||
|
_watched(db, user, 1, [LONG] * 4 + [None] * 3)
|
||||||
|
assert _listed(db, user) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_most_watched_lead_and_a_tie_goes_to_the_one_watched_more_lately(db, make_user):
|
||||||
|
user = make_user("order@example.com")
|
||||||
|
for n in (1, 2, 3):
|
||||||
|
_channel(db, n)
|
||||||
|
_watched(db, user, 1, [LONG] * 6, at=T0 - timedelta(days=100))
|
||||||
|
_watched(db, user, 2, [LONG] * 9, at=T0 - timedelta(days=300))
|
||||||
|
_watched(db, user, 3, [LONG] * 6, at=T0)
|
||||||
|
assert _listed(db, user) == [(_ch(2), 9), (_ch(3), 6), (_ch(1), 6)]
|
||||||
|
last = {ch.id: when for ch, _, when in queries.watched_channels(db, user)}
|
||||||
|
assert last[_ch(3)] == T0 # the last watched of the counted ones
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channel_the_user_follows_blocked_put_out_or_owns_is_not_on_the_list(db, make_user):
|
||||||
|
user = make_user("off@example.com", yt_channel_id=_ch(4))
|
||||||
|
for n in (1, 2, 3, 4, 5):
|
||||||
|
_channel(db, n)
|
||||||
|
_watched(db, user, n, [LONG] * 5)
|
||||||
|
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
|
||||||
|
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(2)))
|
||||||
|
db.commit()
|
||||||
|
service.dismiss_watched_channel(db, user, _ch(3))
|
||||||
|
db.commit()
|
||||||
|
assert _listed(db, user) == [(_ch(5), 5)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_one_users_follows_and_dismissals_leave_anothers_list_alone(db, make_user):
|
||||||
|
me, other = make_user("me@example.com"), make_user("other@example.com")
|
||||||
|
_channel(db, 1)
|
||||||
|
for user in (me, other):
|
||||||
|
_watched(db, user, 1, [LONG] * 5)
|
||||||
|
db.add(Subscription(user_id=other.id, channel_id=_ch(1)))
|
||||||
|
db.commit()
|
||||||
|
service.dismiss_watched_channel(db, other, _ch(1))
|
||||||
|
db.commit()
|
||||||
|
assert _listed(db, me) == [(_ch(1), 5)]
|
||||||
|
assert _listed(db, other) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channel_whose_lengths_are_not_all_known_is_pending_until_they_are(db, make_user):
|
||||||
|
user = make_user("pending@example.com")
|
||||||
|
_channel(db, 1)
|
||||||
|
_watched(db, user, 1, [LONG] * 6, asked=False)
|
||||||
|
_watched(db, user, 2, [LONG] * 4, asked=False) # four watched: it can never make the list
|
||||||
|
assert queries.watched_channels_pending(db, user) == 1
|
||||||
|
assert _listed(db, user) == []
|
||||||
|
ids = [_vid(1, i) for i in range(6)]
|
||||||
|
service.record_takeout_lengths(db, user, ids, dict.fromkeys(ids, LONG))
|
||||||
|
db.commit()
|
||||||
|
assert queries.watched_channels_pending(db, user) == 0
|
||||||
|
assert _listed(db, user) == [(_ch(1), 6)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_new_upload_of_the_history_keeps_the_lengths_already_asked(db, make_user):
|
||||||
|
user = make_user("again@example.com")
|
||||||
|
_channel(db, 1)
|
||||||
|
_watched(db, user, 1, [LONG] * 5)
|
||||||
|
_watched(db, user, 1, [LONG] * 5, at=T0 + timedelta(days=1), asked=False)
|
||||||
|
assert _listed(db, user) == [(_ch(1), 5)]
|
||||||
|
assert queries.takeout_lengths_to_ask(db, user) == []
|
||||||
|
|
||||||
|
|
||||||
|
# --- what the import asks YouTube for ----------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class FakeYouTube:
|
||||||
|
"""Stands in for `YouTubeClient`: answers `videos.list` with the lengths it was given and
|
||||||
|
`channels.list` with a title per channel, and records what it was asked."""
|
||||||
|
|
||||||
|
def __init__(self, lengths: dict[str, int] | None = None, *, fail: bool = False):
|
||||||
|
self.lengths = lengths or {}
|
||||||
|
self.fail = fail
|
||||||
|
self.videos_asked: list[list[str]] = []
|
||||||
|
self.channels_asked: list[list[str]] = []
|
||||||
|
self.actions: list[tuple[int | None, str]] = []
|
||||||
|
|
||||||
|
def __call__(self, db, user):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get_videos(self, ids):
|
||||||
|
self.videos_asked.append(list(ids))
|
||||||
|
self.actions.append((quota._actor_id.get(), quota._action.get()))
|
||||||
|
if self.fail:
|
||||||
|
raise YouTubeError("refused", reason="quotaExceeded", path="videos")
|
||||||
|
return [
|
||||||
|
{"id": v, "contentDetails": {"duration": f"PT{self.lengths[v]}S"}}
|
||||||
|
for v in ids
|
||||||
|
if v in self.lengths
|
||||||
|
]
|
||||||
|
|
||||||
|
def get_channels(self, ids):
|
||||||
|
self.channels_asked.append(list(ids))
|
||||||
|
self.actions.append((quota._actor_id.get(), quota._action.get()))
|
||||||
|
return [{"id": c, "snippet": {"title": f"Named {c[-2:]}"}} for c in ids]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def youtube(monkeypatch, db, make_user):
|
||||||
|
fetcher = make_user("fetcher@example.com")
|
||||||
|
fake = FakeYouTube()
|
||||||
|
monkeypatch.setattr(takeout, "YouTubeClient", fake)
|
||||||
|
monkeypatch.setattr(takeout, "get_service_user", lambda session: fetcher)
|
||||||
|
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: False)
|
||||||
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_import_asks_the_lengths_the_list_needs_and_names_its_channels(db, make_user, youtube):
|
||||||
|
user = make_user("import@example.com")
|
||||||
|
_watched(db, user, 1, [None] * 7, asked=False) # the most watched channel
|
||||||
|
_watched(db, user, 2, [None] * 5, asked=False)
|
||||||
|
_watched(db, user, 3, [None] * 4, asked=False) # too few to ever make it: not asked about
|
||||||
|
youtube.lengths = {_vid(1, i): LONG for i in range(6)} | {_vid(2, i): SHORT for i in range(5)}
|
||||||
|
|
||||||
|
result = takeout.run_import(user.id)
|
||||||
|
|
||||||
|
asked = [_vid(1, i) for i in range(7)] + [_vid(2, i) for i in range(5)]
|
||||||
|
assert youtube.videos_asked == [asked] # the top channel first
|
||||||
|
assert result["lengths_asked"] == 12
|
||||||
|
# Only channel 1 made it (six long; its seventh is gone, channel 2's are short): it is named.
|
||||||
|
assert youtube.channels_asked == [[_ch(1)]]
|
||||||
|
assert result["channels_described"] == 1
|
||||||
|
assert {
|
||||||
|
(user.id, quota.QuotaAction.VIDEOS_LOOKUP),
|
||||||
|
(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||||
|
} == set(youtube.actions)
|
||||||
|
db.expire_all()
|
||||||
|
assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 6)]
|
||||||
|
assert queries.watched_channels_pending(db, user) == 0
|
||||||
|
# Asked once: a second run asks nothing again, the gone video included.
|
||||||
|
youtube.videos_asked.clear()
|
||||||
|
youtube.channels_asked.clear()
|
||||||
|
assert "lengths_asked" not in takeout.run_import(user.id)
|
||||||
|
assert youtube.videos_asked == [] and youtube.channels_asked == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_lookups_stop_at_the_users_daily_share_and_the_rest_wait(
|
||||||
|
db, make_user, youtube, monkeypatch
|
||||||
|
):
|
||||||
|
user = make_user("share@example.com")
|
||||||
|
_watched(db, user, 1, [None] * 8, asked=False)
|
||||||
|
_watched(db, user, 2, [None] * 6, asked=False)
|
||||||
|
monkeypatch.setattr(takeout, "_lookups", takeout.RateLimiter(10, 3600))
|
||||||
|
takeout.run_import(user.id)
|
||||||
|
assert youtube.videos_asked == [[_vid(1, i) for i in range(8)] + [_vid(2, 0), _vid(2, 1)]]
|
||||||
|
db.expire_all()
|
||||||
|
assert queries.watched_channels_pending(db, user) == 1 # channel 2 still has some to ask
|
||||||
|
youtube.videos_asked.clear()
|
||||||
|
takeout.run_import(user.id)
|
||||||
|
assert youtube.videos_asked == [] # the share is spent for today
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("why", ["no_fetcher", "reserve"])
|
||||||
|
def test_nothing_is_asked_without_a_connected_account_or_past_the_reserve(
|
||||||
|
db, make_user, youtube, monkeypatch, why
|
||||||
|
):
|
||||||
|
user = make_user(f"{why}@example.com")
|
||||||
|
_watched(db, user, 1, [None] * 5, asked=False)
|
||||||
|
if why == "no_fetcher":
|
||||||
|
monkeypatch.setattr(takeout, "get_service_user", lambda session: None)
|
||||||
|
else:
|
||||||
|
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: True)
|
||||||
|
assert "lengths_asked" not in takeout.run_import(user.id)
|
||||||
|
assert youtube.videos_asked == []
|
||||||
|
assert queries.watched_channels_pending(db, user) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_youtube_failure_asking_lengths_fails_neither_the_import_nor_its_marks(
|
||||||
|
db, make_user, youtube
|
||||||
|
):
|
||||||
|
user = make_user("fail@example.com")
|
||||||
|
_watched(db, user, 1, [None] * 5, asked=False)
|
||||||
|
youtube.fail = True
|
||||||
|
result = takeout.run_import(user.id)
|
||||||
|
assert "lengths_asked" not in result
|
||||||
|
assert queries.takeout_lengths_to_ask(db, user) == [_vid(1, i) for i in range(5)]
|
||||||
|
db.expire_all()
|
||||||
|
rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
|
||||||
|
assert all(r.looked_up_at is None for r in rows) # left to ask again
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_shorts_job_asks_what_an_import_could_not(db, make_user, youtube):
|
||||||
|
user = make_user("later@example.com")
|
||||||
|
_watched(db, user, 1, [None] * 5, asked=False)
|
||||||
|
youtube.lengths = {_vid(1, i): LONG for i in range(5)}
|
||||||
|
assert takeout.apply_waiting(db)["takeout_lengths_asked"] == 5
|
||||||
|
db.expire_all()
|
||||||
|
assert [n for _, n, _ in queries.watched_channels(db, user)] == [5]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channel_the_user_follows_is_not_asked_about(db, make_user, youtube):
|
||||||
|
user = make_user("follows@example.com")
|
||||||
|
_channel(db, 1)
|
||||||
|
_watched(db, user, 1, [None] * 5, asked=False)
|
||||||
|
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
|
||||||
|
db.commit()
|
||||||
|
assert takeout.apply_waiting(db) == {}
|
||||||
|
assert youtube.videos_asked == []
|
||||||
|
|
||||||
|
|
||||||
|
# --- the channel's watched videos, ready to play ------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_ensure(calls):
|
||||||
|
def ensure(session, yt, ids, *, explore=False):
|
||||||
|
calls.append((list(ids), explore))
|
||||||
|
for vid in ids:
|
||||||
|
if vid.endswith("4"):
|
||||||
|
continue # gone from YouTube since its length was asked
|
||||||
|
session.add(Video(id=vid, channel_id=_ch(1), published_at=T0, via_explore=explore))
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
return ensure
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channels_watched_videos_come_ready_to_play_and_explored(
|
||||||
|
db, make_user, as_user, monkeypatch
|
||||||
|
):
|
||||||
|
user = make_user("play@example.com")
|
||||||
|
_channel(db, 1)
|
||||||
|
_watched(db, user, 1, [LONG, SHORT, LONG, None, LONG, LONG, LONG])
|
||||||
|
db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0)) # already in the catalog
|
||||||
|
db.commit()
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
|
||||||
|
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
|
||||||
|
|
||||||
|
resp = as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos")
|
||||||
|
|
||||||
|
assert resp.status_code == 200, resp.text
|
||||||
|
body = resp.json()
|
||||||
|
# Only the counted ones, newest first; the short one and the gone ones are left out.
|
||||||
|
assert calls == [([_vid(1, 2), _vid(1, 4), _vid(1, 5), _vid(1, 6)], True)]
|
||||||
|
assert [it["id"] for it in body["items"]] == [_vid(1, 0), _vid(1, 2), _vid(1, 5), _vid(1, 6)]
|
||||||
|
assert body["items"][0]["takeout_watched_at"].startswith("2026-05-01")
|
||||||
|
assert body["gone"] == 2 # one YouTube had lost before, one since
|
||||||
|
assert body["ok"] is True
|
||||||
|
db.expire_all()
|
||||||
|
assert db.execute(
|
||||||
|
select(ExploredChannel).where(
|
||||||
|
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == _ch(1)
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
assert (
|
||||||
|
db.get(Channel, _ch(1)).from_explore is True
|
||||||
|
) # nobody follows it: the cleanup's to reclaim
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_blocked_or_unknown_channels_videos_are_refused(db, make_user, as_user):
|
||||||
|
user = make_user("refused@example.com")
|
||||||
|
_channel(db, 1)
|
||||||
|
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
|
||||||
|
db.commit()
|
||||||
|
client = as_user(user)
|
||||||
|
assert client.post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 403
|
||||||
|
assert client.post(f"/api/takeout/channels/{_ch(9)}/videos").status_code == 404
|
||||||
|
|
||||||
|
|
||||||
|
def test_ensure_videos_adds_an_explorations_videos_as_its_own(db):
|
||||||
|
db.add(Channel(id=_ch(1), title="C"))
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
class YT:
|
||||||
|
def get_videos(self, ids):
|
||||||
|
return [{"id": v, "snippet": {"channelId": _ch(1)}} for v in ids]
|
||||||
|
|
||||||
|
sync_videos.ensure_videos(db, YT(), ["exploredvid"], explore=True)
|
||||||
|
sync_videos.ensure_videos(db, YT(), ["organicvid0"])
|
||||||
|
db.expire_all()
|
||||||
|
assert db.get(Video, "exploredvid").via_explore is True
|
||||||
|
assert db.get(Video, "organicvid0").via_explore is False
|
||||||
|
|
||||||
|
|
||||||
|
# --- the routes -----------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_list_route_says_what_it_holds_and_whether_a_history_was_uploaded(
|
||||||
|
db, make_user, as_user
|
||||||
|
):
|
||||||
|
user = make_user("route@example.com")
|
||||||
|
client = as_user(user)
|
||||||
|
assert client.get("/api/takeout/channels").json() == {
|
||||||
|
"imported": False,
|
||||||
|
"pending": 0,
|
||||||
|
"channels": [],
|
||||||
|
}
|
||||||
|
_channel(db, 1, "Some channel")
|
||||||
|
_watched(db, user, 1, [LONG] * 5)
|
||||||
|
_watched(db, user, 2, [None] * 5, asked=False)
|
||||||
|
body = client.get("/api/takeout/channels").json()
|
||||||
|
assert body["imported"] is True and body["pending"] == 1
|
||||||
|
assert [(c["id"], c["title"], c["watched"]) for c in body["channels"]] == [
|
||||||
|
(_ch(1), "Some channel", 5)
|
||||||
|
]
|
||||||
|
assert body["channels"][0]["last_watched_at"].startswith("2026-05-01")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_channel_put_out_of_the_list_comes_back_when_restored(db, make_user, as_user):
|
||||||
|
user = make_user("dismiss@example.com")
|
||||||
|
_channel(db, 1)
|
||||||
|
_watched(db, user, 1, [LONG] * 5)
|
||||||
|
client = as_user(user)
|
||||||
|
url = f"/api/takeout/channels/{_ch(1)}/dismissal"
|
||||||
|
assert client.put(url).json() == {"ok": True, "channel_id": _ch(1)}
|
||||||
|
assert client.put(url).status_code == 200 # again: no error
|
||||||
|
assert client.get("/api/takeout/channels").json()["channels"] == []
|
||||||
|
assert client.delete(url).json() == {"ok": True, "channel_id": _ch(1)}
|
||||||
|
assert [c["id"] for c in client.get("/api/takeout/channels").json()["channels"]] == [_ch(1)]
|
||||||
|
db.expire_all()
|
||||||
|
assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_dismissal_of_an_overlong_channel_id_is_refused_before_the_database(
|
||||||
|
db, make_user, as_user
|
||||||
|
):
|
||||||
|
user = make_user("long@example.com")
|
||||||
|
assert as_user(user).put(f"/api/takeout/channels/{'U' * 33}/dismissal").status_code == 422
|
||||||
Reference in New Issue
Block a user