fix(reader): the watched-channels list keeps its channels, its shares and its retries
- A channel someone follows: the videos its exploration brings in (the list's dialog, the channel page's explore) come in as the catalog's own, not flagged via_explore, which subscribing would never clear and which kept their watched marks waiting for good. - The explore cleanup keeps the row of a channel a Takeout history shows watched (it still reclaims the videos), so a looked-at channel no longer drops off the list. - The Shorts job takes up only users whose list still has lengths to ask or channels to name, by each user's own follows, blocks and dismissals; a naming that failed is tried again, and the list says it is not complete. - The length lookups have a daily share of their own: a large history no longer leaves the export's Watch later file waiting a day. - Only a channel of the user's own history can be put out of the list. - The dialog's videos have a query root of their own: the list's invalidations no longer re-ask (and re-explore) under an open dialog.
This commit is contained in:
@@ -244,6 +244,18 @@ def kept_by_a_reader(video_id: InstrumentedAttribute[str]) -> ColumnElement[bool
|
||||
)
|
||||
|
||||
|
||||
def in_a_watch_history(channel_id: InstrumentedAttribute[str]) -> ColumnElement[bool]:
|
||||
"""A channel someone's uploaded Takeout history shows watched. The "watched but not followed"
|
||||
list names a channel from its catalog row, so the explore cleanup keeps the row of one (its
|
||||
videos it may still reclaim: opening the list's videos brings them back). As a clause, so the
|
||||
cleanup's DELETE stays one statement."""
|
||||
return (
|
||||
select(TakeoutItem.id)
|
||||
.where(TakeoutItem.channel_id == channel_id, TakeoutItem.kind == TakeoutKind.WATCHED)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
def playlisted_by(db: Session, video_ids: list[str]) -> list[tuple[int, str]]:
|
||||
"""(user id, video id) for every user who has one of these videos in a playlist."""
|
||||
return list(
|
||||
@@ -592,17 +604,16 @@ def takeout_lengths_to_ask(db: Session, user: User) -> list[str]:
|
||||
|
||||
|
||||
def watched_channels_pending(db: Session, user: User) -> int:
|
||||
"""How many candidate channels still have a watched video whose length is not known: the
|
||||
channels that may yet join the list."""
|
||||
"""How many channels a later run may still add to the list: the candidates with a watched video
|
||||
whose length is not known, and the ones that made it but the catalog cannot name yet."""
|
||||
cand = _candidate_channels(user)
|
||||
return (
|
||||
db.scalar(
|
||||
select(func.count(func.distinct(TakeoutItem.channel_id)))
|
||||
.join(cand, cand.c.channel_id == TakeoutItem.channel_id)
|
||||
.where(_unfollowed(user), TakeoutItem.looked_up_at.is_(None))
|
||||
)
|
||||
or 0
|
||||
)
|
||||
unasked = db.execute(
|
||||
select(TakeoutItem.channel_id)
|
||||
.join(cand, cand.c.channel_id == TakeoutItem.channel_id)
|
||||
.where(_unfollowed(user), TakeoutItem.looked_up_at.is_(None))
|
||||
.distinct()
|
||||
).scalars()
|
||||
return len(set(unasked) | set(watched_channels_to_describe(db, user)))
|
||||
|
||||
|
||||
def watched_channels_to_describe(db: Session, user: User) -> list[str]:
|
||||
@@ -655,6 +666,23 @@ def has_watch_history(db: Session, user: User) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def watched_in_history(db: Session, user: User, channel_id: str) -> bool:
|
||||
"""Whether the user's Takeout history shows them watching this channel at all."""
|
||||
return bool(
|
||||
db.scalar(
|
||||
select(
|
||||
select(TakeoutItem.id)
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||
TakeoutItem.channel_id == channel_id,
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def watched_channel_videos(
|
||||
db: Session, user: User, channel_id: str
|
||||
) -> tuple[list[tuple[str, datetime]], int]:
|
||||
@@ -680,18 +708,19 @@ def watched_channel_videos(
|
||||
return counted, gone
|
||||
|
||||
|
||||
def takeout_users_with_lengths_to_ask(db: Session) -> list[int]:
|
||||
"""Every user with a channel of enough watched videos not all asked about yet. It does not ask
|
||||
whether they follow the channel: that is each user's own question, which
|
||||
`takeout_lengths_to_ask` answers (and an empty answer costs a run nothing)."""
|
||||
def takeout_users_with_list_work(db: Session) -> list[int]:
|
||||
"""Every user whose "watched but not followed" list a run can still add to: lengths to ask, or
|
||||
channels to name. Asked of each user whose history holds a channel of enough watched videos,
|
||||
since what is off a list (followed, blocked, put out, their own) is that user's own — a row of
|
||||
one is never asked about, and would otherwise bring the user back on every run."""
|
||||
per_channel = (
|
||||
select(TakeoutItem.user_id)
|
||||
.where(TakeoutItem.kind == TakeoutKind.WATCHED, TakeoutItem.channel_id.is_not(None))
|
||||
.group_by(TakeoutItem.user_id, TakeoutItem.channel_id)
|
||||
.having(
|
||||
func.count() >= WATCHED_CHANNEL_MIN_VIDEOS,
|
||||
func.count().filter(TakeoutItem.looked_up_at.is_(None)) > 0,
|
||||
)
|
||||
.having(func.count() >= WATCHED_CHANNEL_MIN_VIDEOS)
|
||||
.subquery()
|
||||
)
|
||||
return list(db.execute(select(per_channel.c.user_id).distinct()).scalars())
|
||||
users = db.execute(
|
||||
select(User).where(User.id.in_(select(per_channel.c.user_id))).order_by(User.id)
|
||||
).scalars()
|
||||
return [user.id for user in users if watched_channels_pending(db, user)]
|
||||
|
||||
@@ -322,6 +322,10 @@ def stage(db: Session, user: User, parsed: Parsed) -> None:
|
||||
# How many unknown Watch later videos one user may have looked up on the shared API budget in a day:
|
||||
# two full Watch later lists (two units a hundred ids). The rest wait for the next day's runs.
|
||||
_lookups = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600)
|
||||
# How many watched videos' lengths one user may ask about in a day for the "watched but not followed"
|
||||
# list (a unit a fifty). A share of its own: a large history asks for thousands, and on the Watch
|
||||
# later share it would leave the export's Watch later file waiting a day behind it.
|
||||
_lengths = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600)
|
||||
|
||||
|
||||
def _put_in_watch_later(db: Session, user: User) -> dict:
|
||||
@@ -356,8 +360,8 @@ def _put_in_watch_later(db: Session, user: User) -> dict:
|
||||
|
||||
def _ask_lengths(db: Session, user: User) -> dict:
|
||||
"""Ask YouTube how long the user's watched videos are that the "watched but not followed" list
|
||||
still needs, and keep the answer — on the same terms and the same daily share as the Watch
|
||||
later lookups. What is not asked yet (no YouTube-connected account, the reserve reached, the
|
||||
still needs, and keep the answer — on the same terms as the Watch later lookups, on a daily
|
||||
share of its own. What is not asked yet (no YouTube-connected account, the reserve reached, the
|
||||
share spent) waits for a later run, and the list says it is not complete. A YouTube failure is
|
||||
logged and left for a later run too: it is no reason to fail the import it follows. The caller
|
||||
commits."""
|
||||
@@ -365,7 +369,7 @@ def _ask_lengths(db: Session, user: User) -> dict:
|
||||
fetcher = get_service_user(db) if ask else None
|
||||
if fetcher is None or quota.backfill_reserve_reached():
|
||||
return {}
|
||||
ask = ask[: int(_lookups.room(str(user.id)))]
|
||||
ask = ask[: int(_lengths.room(str(user.id)))]
|
||||
if not ask:
|
||||
return {}
|
||||
try:
|
||||
@@ -379,7 +383,7 @@ def _ask_lengths(db: Session, user: User) -> dict:
|
||||
extra=logs.event("takeout.lengths.failed", user_id=user.id, reason=exc.reason),
|
||||
)
|
||||
return {}
|
||||
_lookups.allow(str(user.id), cost=len(ask))
|
||||
_lengths.allow(str(user.id), cost=len(ask))
|
||||
lengths = {
|
||||
it["id"]: parse_iso8601_duration(it.get("contentDetails", {}).get("duration"))
|
||||
for it in items
|
||||
@@ -453,7 +457,7 @@ def apply_waiting(db: Session) -> dict:
|
||||
marked = added = asked = failed = 0
|
||||
ready = set(queries.takeout_users_ready(db))
|
||||
later = set(queries.takeout_users_watch_later_waiting(db))
|
||||
lengths = set(queries.takeout_users_with_lengths_to_ask(db))
|
||||
lengths = set(queries.takeout_users_with_list_work(db))
|
||||
for user_id in sorted(ready | later | lengths):
|
||||
user = db.get(User, user_id)
|
||||
if user is None:
|
||||
|
||||
Reference in New Issue
Block a user