Compare commits
15
Commits
bf5ceec8a1
...
v0.111.0
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4695e1a62f | ||
|
|
1adf59ea1b | ||
|
|
bc8fe2cc6c | ||
|
|
33dd9ff054 | ||
|
|
b5f9cf14db | ||
|
|
bd2340c204 | ||
|
|
84af92aa29 | ||
|
|
420baff463 | ||
|
|
9c5f81fd94 | ||
|
|
9f580fd8c8 | ||
|
|
e0b02a0c14 | ||
|
|
3553eac8c4 | ||
|
|
5d9952ab1e | ||
|
|
f486a69ef2 | ||
|
|
7b8505af95 |
@@ -7,6 +7,28 @@ edit the release notes and re-run the generator.
|
||||
|
||||
Upgrading, pinning a version and rolling back: [docs/self-hosting.md](docs/self-hosting.md#upgrading).
|
||||
|
||||
## 0.111.0 — 2026-10-24
|
||||
|
||||
### New
|
||||
|
||||
- Channels › Discover from your history: the channels your uploaded YouTube history shows you watching often (five videos of five minutes or more) but not following.
|
||||
- Follow one, put it out of the list (with Undo), or open the videos you watched there and play them one by one or all in a row.
|
||||
|
||||
### Fixed
|
||||
|
||||
- A channel someone explored and then followed on YouTube is now part of the shared Library like any followed channel, and so are its videos; before, they could stay hidden and their watched marks never applied.
|
||||
|
||||
## 0.110.0 — 2026-10-23
|
||||
|
||||
### New
|
||||
|
||||
- YouTube history: upload your Google Takeout export in Settings › Account. What you watched on YouTube is marked watched here (never a Short, never over a state you set), and your YouTube Watch later list joins yours.
|
||||
- A video the catalog does not know yet is remembered and marked once it arrives. The import's outcome reaches your inbox, and admins see each run on the Scheduler page.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Scheduler: a channel's full-history backfill asked for again while one is running now runs after it, instead of being silently dropped.
|
||||
|
||||
## 0.109.2 — 2026-10-23
|
||||
|
||||
### Fixed
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""A user's Google Takeout export: what they watched on YouTube, and their Watch later list
|
||||
|
||||
Revision ID: 0113_takeout_items
|
||||
Revises: 0112_one_watch_later
|
||||
Create Date: 2026-10-23
|
||||
|
||||
One row per video and kind. A video the catalog does not know yet waits with no outcome, so the id
|
||||
is not a foreign key; the partial index is what a newly known video looks its waiting rows up by.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0113_takeout_items"
|
||||
down_revision: str | None = "0112_one_watch_later"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"takeout_items",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("kind", sa.String(12), nullable=False),
|
||||
sa.Column("video_id", sa.String(16), nullable=False),
|
||||
sa.Column("channel_id", sa.String(32), nullable=True),
|
||||
sa.Column("at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("outcome", sa.String(12), nullable=True),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False
|
||||
),
|
||||
sa.UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"),
|
||||
sa.CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_takeout_items_waiting",
|
||||
"takeout_items",
|
||||
["video_id"],
|
||||
postgresql_where=sa.text("outcome IS NULL"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_takeout_items_waiting", table_name="takeout_items")
|
||||
op.drop_table("takeout_items")
|
||||
@@ -0,0 +1,44 @@
|
||||
"""When a user last took a video back to "new"
|
||||
|
||||
Revision ID: 0114_video_unmarks
|
||||
Revises: 0113_takeout_items
|
||||
Create Date: 2026-10-23
|
||||
|
||||
Un-marking or resetting a video leaves no `video_states` row, so this is what tells an import of the
|
||||
user's YouTube history a video they un-marked here from one never touched.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0114_video_unmarks"
|
||||
down_revision: str | None = "0113_takeout_items"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"video_unmarks",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"video_id",
|
||||
sa.String(16),
|
||||
sa.ForeignKey("videos.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("user_id", "video_id", name="uq_video_unmark"),
|
||||
)
|
||||
op.create_index("ix_video_unmarks_video_id", "video_unmarks", ["video_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_video_unmarks_video_id", table_name="video_unmarks")
|
||||
op.drop_table("video_unmarks")
|
||||
@@ -0,0 +1,49 @@
|
||||
"""A watched Takeout video's length, and the channels a user put out of their list
|
||||
|
||||
Revision ID: 0115_takeout_channels
|
||||
Revises: 0114_video_unmarks
|
||||
Create Date: 2026-10-24
|
||||
|
||||
The "channels you watch but do not follow" list counts a user's watched videos over five minutes
|
||||
by channel. The length is asked of YouTube for that list and kept on the Takeout row; a channel the
|
||||
user does not want on the list is kept apart.
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0115_takeout_channels"
|
||||
down_revision: str | None = "0114_video_unmarks"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"takeout_items", sa.Column("looked_up_at", sa.DateTime(timezone=True), nullable=True)
|
||||
)
|
||||
op.add_column("takeout_items", sa.Column("duration_seconds", sa.Integer(), nullable=True))
|
||||
op.create_table(
|
||||
"takeout_channel_dismissals",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"user_id",
|
||||
sa.Integer(),
|
||||
sa.ForeignKey("users.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("channel_id", sa.String(32), nullable=False),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(),
|
||||
nullable=False,
|
||||
),
|
||||
sa.UniqueConstraint("user_id", "channel_id", name="uq_takeout_channel_dismissal"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("takeout_channel_dismissals")
|
||||
op.drop_column("takeout_items", "duration_seconds")
|
||||
op.drop_column("takeout_items", "looked_up_at")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""An explored channel someone follows is the catalog's own, and so are its videos
|
||||
|
||||
Revision ID: 0116_followed_explored_channels
|
||||
Revises: 0115_takeout_channels
|
||||
Create Date: 2026-10-24
|
||||
|
||||
A follow pulled in from YouTube (the subscription import) left an explored channel flagged as an
|
||||
exploration's, and a subscribe landing while an exploration's videos came in left those videos
|
||||
flagged on a channel that no longer was one. Both are held in the code now; this clears what they
|
||||
left: the channel of a subscription is not an exploration's, and no video is an exploration's on a
|
||||
channel that is not one. Data only, so the downgrade has nothing to undo.
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0116_followed_explored_channels"
|
||||
down_revision: str | None = "0115_takeout_channels"
|
||||
branch_labels: str | None = None
|
||||
depends_on: str | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"UPDATE channels SET from_explore = false WHERE from_explore"
|
||||
" AND EXISTS (SELECT 1 FROM subscriptions s WHERE s.channel_id = channels.id)"
|
||||
)
|
||||
op.execute(
|
||||
"UPDATE videos SET via_explore = false WHERE via_explore"
|
||||
" AND channel_id IN (SELECT id FROM channels WHERE NOT from_explore)"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -72,6 +72,13 @@ DEFAULT_MAX_BODY_BYTES = 1024 * 1024
|
||||
# A route belongs here ONLY IF ITS OWN CAP IS LARGER THAN THE DEFAULT. That is why
|
||||
# `POST /api/diagnostics/events` is absent: its `capped_body` limit is 512 KiB, already below the
|
||||
# default, so listing it could only widen what it accepts.
|
||||
# A Google Takeout file (`reader.takeout`). A real watch history measured 3.7 MB for 9500 views, ~390
|
||||
# bytes a view, so 32 MiB holds ~86k views. YouTube keeps the history 36 months by default on an
|
||||
# account made since mid-2020 and for good on an older one, so a heavy viewer's can pass 16 MiB. The
|
||||
# file's text is decoded a window at a time and read an entry at a time (`takeout._json_list`), so
|
||||
# it costs its bytes and a window, not a tree of it; the route takes two at once across every user.
|
||||
TAKEOUT_MAX_BYTES = 32 * 1024 * 1024
|
||||
|
||||
ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
||||
# Ingesting a local file. The body IS the file now — no multipart envelope to allow for —
|
||||
# so this ceiling and the route's own cap count exactly the same bytes. `sysconfig`'s
|
||||
@@ -95,6 +102,7 @@ ROUTE_MAX_BODY_BYTES: dict[tuple[str, str], int] = {
|
||||
# sending it at all, off the `upload_max_bytes` that `GET /api/files/usage` reports.
|
||||
("POST", "/api/files/upload"): settings.download_upload_max_bytes,
|
||||
("POST", "/api/debug-sessions/{sid}/screenshots"): debugsession.SCREENSHOT_MAX_BYTES,
|
||||
("POST", "/api/takeout"): TAKEOUT_MAX_BYTES,
|
||||
}
|
||||
|
||||
# The refusal, byte-for-byte what `capped_body`'s `HTTPException(413, {"code": "too_large"})` renders
|
||||
|
||||
@@ -71,10 +71,12 @@ class Channel(Base, TimestampMixin):
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
# True when this channel exists only because a user EXPLORED it (opened its page without
|
||||
# subscribing) — an ephemeral, per-explorer addition. The cleanup job hard-deletes such a
|
||||
# channel (and its via_explore videos) once nobody keeps it (no subscription, no live
|
||||
# ExploredChannel row, no per-user state) after a grace period. Cleared when someone
|
||||
# subscribes (the "keep" signal promotes it to organic).
|
||||
# subscribing) — an ephemeral, per-explorer addition. After a grace period the cleanup job
|
||||
# hard-deletes its via_explore videos nobody keeps (no subscription, no live ExploredChannel
|
||||
# row, no per-user state), and then the channel row itself once it holds no videos — unless a
|
||||
# Takeout watch history shows it watched: the "watched but not followed" list names it from
|
||||
# that row. Cleared when someone subscribes, here or on YouTube (the "keep" signal promotes it
|
||||
# to organic).
|
||||
from_explore: Mapped[bool] = mapped_column(
|
||||
Boolean, default=False, server_default="false", index=True
|
||||
)
|
||||
@@ -235,8 +237,9 @@ class ExploredChannel(Base, TimestampMixin):
|
||||
Exploring an un-subscribed channel ingests its recent uploads (flagged via_explore) so the
|
||||
user can browse them, but keeps them out of everyone else's catalog. This row (a) gates the
|
||||
explorer's access to those ephemeral videos in the feed query and (b) drives the grace-period
|
||||
purge: a from_explore channel with no live ExploredChannel row (and no subscription / no
|
||||
per-user state on its videos) is hard-deleted by the cleanup job. Subscribing is the "keep"
|
||||
purge: of a from_explore channel with no live ExploredChannel row (and no subscription) the
|
||||
cleanup job hard-deletes the videos without per-user state, then the channel once it holds none
|
||||
and no Takeout watch history shows it watched. Subscribing is the "keep"
|
||||
signal — it promotes the channel to organic and these rows become irrelevant. `created_at`
|
||||
(from TimestampMixin) is the grace clock; a re-visit refreshes it."""
|
||||
|
||||
|
||||
@@ -96,13 +96,24 @@ def record_subscription(db: Session, user: User, channel: Channel, yt_subscripti
|
||||
subscribed_at=datetime.now(timezone.utc),
|
||||
)
|
||||
)
|
||||
if channel.from_explore:
|
||||
channel.from_explore = False
|
||||
db.execute(
|
||||
update(Video)
|
||||
.where(Video.channel_id == channel.id, Video.via_explore.is_(True))
|
||||
.values(via_explore=False)
|
||||
)
|
||||
db.flush()
|
||||
settle_explored(db, channel)
|
||||
|
||||
|
||||
def settle_explored(db: Session, channel: Channel) -> None:
|
||||
"""Hold the rule that a channel's videos are an exploration's only while nobody follows it,
|
||||
decided by who follows it now, not by the flag held in memory: a subscribe in another request
|
||||
may have cleared that while YouTube answered this one, and the videos that came in meanwhile
|
||||
would stay flagged. A channel someone follows, however the follow came (here or pulled in from
|
||||
YouTube), is the catalog's own, and so are its videos."""
|
||||
if channel.from_explore and not queries.followed(db, channel.id):
|
||||
return
|
||||
channel.from_explore = False
|
||||
db.execute(
|
||||
update(Video)
|
||||
.where(Video.channel_id == channel.id, Video.via_explore.is_(True))
|
||||
.values(via_explore=False)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
|
||||
@@ -119,6 +119,16 @@ def recent_runs(db: Session, jobs: list[str], per_job: int) -> dict[str, list[Jo
|
||||
return out
|
||||
|
||||
|
||||
def last_run(db: Session, job: str, target: str) -> JobRun | None:
|
||||
"""The newest run of an ad-hoc `job` on `target` (a user's own import), or None."""
|
||||
return db.execute(
|
||||
select(JobRun)
|
||||
.where(JobRun.job == job, JobRun.target == target)
|
||||
.order_by(JobRun.started_at.desc(), JobRun.id.desc())
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def gc(db: Session, retention_days: int) -> int:
|
||||
"""Delete finished runs older than `retention_days` (0 = keep forever). A run still going is
|
||||
never reaped. The caller commits."""
|
||||
|
||||
@@ -42,6 +42,14 @@ class RateLimiter:
|
||||
with self._lock:
|
||||
return sum(c for t, c in self._hits.get(key, []) if t > cutoff) >= self.max_events
|
||||
|
||||
def room(self, key: str) -> float:
|
||||
"""How much cost ``key`` may still spend in the current window, WITHOUT counting anything —
|
||||
for work counted only once it has succeeded: ask it first, ``allow`` what was done."""
|
||||
cutoff = time.monotonic() - self.window
|
||||
with self._lock:
|
||||
used = sum(c for t, c in self._hits.get(key, []) if t > cutoff)
|
||||
return max(0.0, self.max_events - used)
|
||||
|
||||
def allow(self, key: str, cost: float = 1) -> bool:
|
||||
"""Record an attempt costing ``cost`` for ``key`` and return whether it is within the limit.
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ and the comfort shelf on Home. Only this unit writes them; another unit reads th
|
||||
edge, or asks this unit's service."""
|
||||
|
||||
from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
@@ -84,6 +85,21 @@ class VideoState(Base, UpdatedAtMixin):
|
||||
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class VideoUnmark(Base):
|
||||
"""When the user last took a video back to "new" here — un-marked it, or reset it. That leaves
|
||||
no `VideoState` (its absence is "new"), so without this row a later import of what they watched
|
||||
on YouTube could not tell a video they un-marked from one never touched, and marked it watched
|
||||
again. Kept by user and video; the video's own deletion takes it."""
|
||||
|
||||
__tablename__ = "video_unmarks"
|
||||
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_video_unmark"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
||||
at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class Playlist(Base, TimestampMixin, UpdatedAtMixin):
|
||||
"""A per-user named, ordered collection of videos from the shared catalog.
|
||||
|
||||
@@ -213,3 +229,70 @@ class ComfortPin(Base, TimestampMixin):
|
||||
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
|
||||
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
|
||||
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
||||
|
||||
|
||||
class TakeoutKind(StrEnum):
|
||||
"""What a Google Takeout row says the user did with a video."""
|
||||
|
||||
WATCHED = "watched" # history/watch-history.json
|
||||
WATCH_LATER = "watch_later" # playlists/Watch later-videos.csv
|
||||
|
||||
|
||||
class TakeoutOutcome(StrEnum):
|
||||
"""What the import did with a row; NULL = still waiting for its video to be known (and, for a
|
||||
watched one, probed for Shorts)."""
|
||||
|
||||
WATCHED = "watched" # the video is marked watched at the Takeout time
|
||||
KEPT = "kept" # the state the user set here (watched, hidden, or changed later) stays
|
||||
SHORT = "short" # a Short: swiping one past is not watching it
|
||||
ADDED = "added" # added to Watch later
|
||||
PRESENT = "present" # already in Watch later
|
||||
MISSING = "missing" # YouTube no longer has the video
|
||||
|
||||
|
||||
class TakeoutItem(Base, TimestampMixin):
|
||||
"""One video from a user's Google Takeout export: watched (with when) or in their YouTube Watch
|
||||
later list (with when it was added). The rows are both the import job's input and its memory: a
|
||||
video the catalog does not know yet waits here, with no outcome, until it enters the catalog.
|
||||
The id is not a foreign key for that reason. A watched row's channel and length feed the
|
||||
"channels you watch but do not follow" list: the length is asked of YouTube for that list alone
|
||||
and kept here, so its videos need not enter the shared catalog to be counted."""
|
||||
|
||||
__tablename__ = "takeout_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"),
|
||||
CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"),
|
||||
# The waiting rows, by video: what a newly known video looks up.
|
||||
Index(
|
||||
"ix_takeout_items_waiting",
|
||||
"video_id",
|
||||
postgresql_where=text("outcome IS NULL"),
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
kind: Mapped[str] = mapped_column(String(12))
|
||||
video_id: Mapped[str] = mapped_column(String(16))
|
||||
channel_id: Mapped[str | None] = mapped_column(String(32))
|
||||
# Watched: the last time in the export; Watch later: when it was added.
|
||||
at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
outcome: Mapped[str | None] = mapped_column(String(12))
|
||||
# When YouTube was asked for the video's length (NULL = not yet), and the length it gave; a
|
||||
# video it no longer has keeps a NULL length.
|
||||
looked_up_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
duration_seconds: Mapped[int | None] = mapped_column(Integer)
|
||||
|
||||
|
||||
class TakeoutChannelDismissal(Base, TimestampMixin):
|
||||
"""A channel the user put out of their "watched but not followed" list. The id is not a foreign
|
||||
key: the channel may never have entered the catalog, or may leave it, and the choice stays."""
|
||||
|
||||
__tablename__ = "takeout_channel_dismissals"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "channel_id", name="uq_takeout_channel_dismissal"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
||||
channel_id: Mapped[str] = mapped_column(String(32))
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Reader's reads: a user's labels, saved views, watch state and playlists, for the HTTP layer and
|
||||
for the other units that ask about them.
|
||||
"""Reader's reads: a user's labels, saved views, watch state, playlists and Takeout import, for the HTTP
|
||||
layer and for the other units that ask about them.
|
||||
|
||||
Every read of a user's row goes through `userscope`, so "someone else's row" is the same answer as
|
||||
a missing one. Reader reads Catalog's videos along its declared edge (a playlist's cover and
|
||||
@@ -7,14 +7,28 @@ length, whether a video exists); another unit asks the questions at the end of t
|
||||
than joining Reader's tables itself.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ColumnElement, and_, func, or_, select
|
||||
from sqlalchemy.orm import InstrumentedAttribute, Session, aliased
|
||||
|
||||
from app.collation import collated
|
||||
from app.feedquery import feed_columns, serialize
|
||||
from app.models import User
|
||||
from app.catalog.models import Channel, Subscription, Video
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
||||
from app.catalog.models import BlockedChannel, Channel, Subscription, Video
|
||||
from app.reader.models import (
|
||||
ChannelTag,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SavedView,
|
||||
Tag,
|
||||
TakeoutChannelDismissal,
|
||||
TakeoutItem,
|
||||
TakeoutKind,
|
||||
VideoState,
|
||||
VideoUnmark,
|
||||
)
|
||||
from app.userscope import owned_by, owned_row
|
||||
|
||||
# --- catalog, read along the edge ------------------------------------------------------------
|
||||
@@ -230,6 +244,18 @@ def kept_by_a_reader(video_id: InstrumentedAttribute[str]) -> ColumnElement[bool
|
||||
)
|
||||
|
||||
|
||||
def in_a_watch_history(channel_id: InstrumentedAttribute[str]) -> ColumnElement[bool]:
|
||||
"""A channel someone's uploaded Takeout history shows watched. The "watched but not followed"
|
||||
list names a channel from its catalog row, so the explore cleanup keeps the row of one (its
|
||||
videos it may still reclaim: opening the list's videos brings them back). As a clause, so the
|
||||
cleanup's DELETE stays one statement."""
|
||||
return (
|
||||
select(TakeoutItem.id)
|
||||
.where(TakeoutItem.channel_id == channel_id, TakeoutItem.kind == TakeoutKind.WATCHED)
|
||||
.exists()
|
||||
)
|
||||
|
||||
|
||||
def playlisted_by(db: Session, video_ids: list[str]) -> list[tuple[int, str]]:
|
||||
"""(user id, video id) for every user who has one of these videos in a playlist."""
|
||||
return list(
|
||||
@@ -359,3 +385,355 @@ def user_tag_link(db: Session, user: User, channel_id: str, tag_id: int) -> Chan
|
||||
owned_by(ChannelTag.user_id, user),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
# --- Google Takeout --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _settled_here() -> ColumnElement[bool]:
|
||||
"""A video a waiting "watched" row may be marked on: one the catalog keeps of its own, not a
|
||||
discovery video only a search or an explore brought in. A mark is a `VideoState`, which the
|
||||
cleanup jobs read as a reader keeping it (`kept_by_a_reader`): on one of those it would keep the
|
||||
video, and its explored channel, for good. Such a row waits until the video is the catalog's own
|
||||
(its channel subscribed to) — or is gone."""
|
||||
subscribed = select(Subscription.id).where(Subscription.channel_id == Video.channel_id).exists()
|
||||
return and_(Video.via_explore.is_(False), or_(Video.via_search.is_(False), subscribed))
|
||||
|
||||
|
||||
def takeout_watched_ready(db: Session, user: User) -> list[tuple[int, str, datetime, bool]]:
|
||||
"""The user's waiting "watched" rows whose video the catalog knows and has probed for Shorts:
|
||||
(row id, video id, Takeout time, is Short)."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.id, TakeoutItem.video_id, TakeoutItem.at, Video.is_short)
|
||||
.join(Video, Video.id == TakeoutItem.video_id)
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||
TakeoutItem.outcome.is_(None),
|
||||
Video.shorts_probed,
|
||||
_settled_here(),
|
||||
)
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def takeout_waiting(db: Session, user: User, kind: TakeoutKind) -> list[tuple[int, str]]:
|
||||
"""The user's rows of `kind` the import has not settled, oldest first: (row id, video id)."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.id, TakeoutItem.video_id)
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == kind,
|
||||
TakeoutItem.outcome.is_(None),
|
||||
)
|
||||
.order_by(TakeoutItem.at, TakeoutItem.id)
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
# Ids per IN list: a list is bound one parameter an id, and a statement takes at most 65,535.
|
||||
_IN_CHUNK = 5000
|
||||
|
||||
|
||||
def known_video_ids(db: Session, ids: list[str]) -> set[str]:
|
||||
"""Which of `ids` the catalog holds."""
|
||||
known: set[str] = set()
|
||||
for start in range(0, len(ids), _IN_CHUNK):
|
||||
chunk = ids[start : start + _IN_CHUNK]
|
||||
known.update(db.execute(select(Video.id).where(Video.id.in_(chunk))).scalars())
|
||||
return known
|
||||
|
||||
|
||||
def unmarked_at(db: Session, user: User, ids: list[str]) -> dict[str, datetime]:
|
||||
"""When the user last took each of `ids` back to "new" here, for those they ever did."""
|
||||
out: dict[str, datetime] = {}
|
||||
for start in range(0, len(ids), _IN_CHUNK):
|
||||
rows = db.execute(
|
||||
select(VideoUnmark.video_id, VideoUnmark.at).where(
|
||||
owned_by(VideoUnmark.user_id, user),
|
||||
VideoUnmark.video_id.in_(ids[start : start + _IN_CHUNK]),
|
||||
)
|
||||
).tuples()
|
||||
out.update((vid, at) for vid, at in rows)
|
||||
return out
|
||||
|
||||
|
||||
def in_playlist(db: Session, playlist_id: int, ids: list[str]) -> set[str]:
|
||||
"""Which of `ids` the playlist already holds."""
|
||||
held: set[str] = set()
|
||||
for start in range(0, len(ids), _IN_CHUNK):
|
||||
chunk = ids[start : start + _IN_CHUNK]
|
||||
held.update(
|
||||
db.execute(
|
||||
select(PlaylistItem.video_id).where(
|
||||
PlaylistItem.playlist_id == playlist_id, PlaylistItem.video_id.in_(chunk)
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
return held
|
||||
|
||||
|
||||
def takeout_rows_besides(
|
||||
db: Session, user: User, kind: TakeoutKind, video_ids: Iterable[str]
|
||||
) -> int:
|
||||
"""How many Takeout rows the user keeps other than `video_ids` of `kind`: what an upload of
|
||||
those videos would be added to."""
|
||||
total = (
|
||||
db.scalar(select(func.count(TakeoutItem.id)).where(owned_by(TakeoutItem.user_id, user)))
|
||||
or 0
|
||||
)
|
||||
ids = list(video_ids)
|
||||
for start in range(0, len(ids), _IN_CHUNK):
|
||||
total -= (
|
||||
db.scalar(
|
||||
select(func.count(TakeoutItem.id)).where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == kind,
|
||||
TakeoutItem.video_id.in_(ids[start : start + _IN_CHUNK]),
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return total
|
||||
|
||||
|
||||
def takeout_outcomes(db: Session, user: User) -> dict[tuple[str, str | None], int]:
|
||||
"""How many of the user's rows each kind and outcome holds (None = waiting)."""
|
||||
rows = db.execute(
|
||||
select(TakeoutItem.kind, TakeoutItem.outcome, func.count())
|
||||
.where(owned_by(TakeoutItem.user_id, user))
|
||||
.group_by(TakeoutItem.kind, TakeoutItem.outcome)
|
||||
).all()
|
||||
return {(kind, outcome): n for kind, outcome, n in rows}
|
||||
|
||||
|
||||
def takeout_summary(db: Session, user: User) -> dict[str, dict[str, int]]:
|
||||
"""The user's kept Takeout videos by kind and outcome, for their settings page: `{"watched":
|
||||
{"watched": n, "short": n, "waiting": n, …}, "watch_later": {…}}` — the import's outcome words,
|
||||
"waiting" for a video not known yet."""
|
||||
out: dict[str, dict[str, int]] = {kind.value: {} for kind in TakeoutKind}
|
||||
for (kind, outcome), n in takeout_outcomes(db, user).items():
|
||||
out[kind][outcome or "waiting"] = n
|
||||
return out
|
||||
|
||||
|
||||
def takeout_users_ready(db: Session) -> list[int]:
|
||||
"""Every user with a waiting "watched" row whose video is now known and probed for Shorts: the
|
||||
rows a later run of the Shorts probe made ready."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.user_id)
|
||||
.join(Video, Video.id == TakeoutItem.video_id)
|
||||
.where(
|
||||
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||
TakeoutItem.outcome.is_(None),
|
||||
Video.shorts_probed,
|
||||
_settled_here(),
|
||||
)
|
||||
.distinct()
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def takeout_users_watch_later_waiting(db: Session) -> list[int]:
|
||||
"""Every user with a Watch later row a run could not put in yet."""
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.user_id)
|
||||
.where(TakeoutItem.kind == TakeoutKind.WATCH_LATER, TakeoutItem.outcome.is_(None))
|
||||
.distinct()
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
# --- channels watched but not followed ---------------------------------------------------------
|
||||
|
||||
# A channel joins the list with this many watched videos of at least this length. A Short is three
|
||||
# minutes at most, so the length alone keeps them out, and a video under five minutes says too
|
||||
# little of what a channel is about to count towards it.
|
||||
WATCHED_CHANNEL_MIN_VIDEOS = 5
|
||||
WATCHED_CHANNEL_MIN_SECONDS = 300
|
||||
|
||||
|
||||
def _unfollowed(user: User) -> ColumnElement[bool]:
|
||||
"""A watched row whose channel the user neither follows, blocked, put out of the list, nor is."""
|
||||
subscribed = select(Subscription.channel_id).where(owned_by(Subscription.user_id, user))
|
||||
blocked = select(BlockedChannel.channel_id).where(owned_by(BlockedChannel.user_id, user))
|
||||
dismissed = select(TakeoutChannelDismissal.channel_id).where(
|
||||
owned_by(TakeoutChannelDismissal.user_id, user)
|
||||
)
|
||||
clauses = [
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||
TakeoutItem.channel_id.is_not(None),
|
||||
TakeoutItem.channel_id.not_in(subscribed),
|
||||
TakeoutItem.channel_id.not_in(blocked),
|
||||
TakeoutItem.channel_id.not_in(dismissed),
|
||||
]
|
||||
if user.yt_channel_id:
|
||||
clauses.append(TakeoutItem.channel_id != user.yt_channel_id)
|
||||
return and_(*clauses)
|
||||
|
||||
|
||||
def _candidate_channels(user: User):
|
||||
"""The unfollowed channels with enough watched videos to make the list once their lengths are
|
||||
known: (channel id, how many watched)."""
|
||||
return (
|
||||
select(TakeoutItem.channel_id, func.count().label("watched"))
|
||||
.where(_unfollowed(user))
|
||||
.group_by(TakeoutItem.channel_id)
|
||||
.having(func.count() >= WATCHED_CHANNEL_MIN_VIDEOS)
|
||||
.subquery()
|
||||
)
|
||||
|
||||
|
||||
def takeout_lengths_to_ask(db: Session, user: User) -> list[str]:
|
||||
"""The watched videos whose length the list still needs: those of the candidate channels not
|
||||
asked about yet, the most watched channels first, so a lookup cut short fills the top."""
|
||||
cand = _candidate_channels(user)
|
||||
return list(
|
||||
db.execute(
|
||||
select(TakeoutItem.video_id)
|
||||
.join(cand, cand.c.channel_id == TakeoutItem.channel_id)
|
||||
.where(_unfollowed(user), TakeoutItem.looked_up_at.is_(None))
|
||||
.order_by(cand.c.watched.desc(), TakeoutItem.channel_id, TakeoutItem.at.desc())
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def watched_channels_pending(db: Session, user: User) -> int:
|
||||
"""How many channels a later run may still add to the list: the candidates with a watched video
|
||||
whose length is not known, and the ones that made it but have no catalog row to be listed from
|
||||
yet. One the catalog holds without details is listed already (by the name it came in with), so
|
||||
it is not one to come."""
|
||||
cand = _candidate_channels(user)
|
||||
unasked = db.execute(
|
||||
select(TakeoutItem.channel_id)
|
||||
.join(cand, cand.c.channel_id == TakeoutItem.channel_id)
|
||||
.where(_unfollowed(user), TakeoutItem.looked_up_at.is_(None))
|
||||
.distinct()
|
||||
).scalars()
|
||||
counted = _counted_channels(user)
|
||||
unlisted = db.execute(
|
||||
select(counted.c.channel_id)
|
||||
.outerjoin(Channel, Channel.id == counted.c.channel_id)
|
||||
.where(Channel.id.is_(None))
|
||||
).scalars()
|
||||
return len(set(unasked) | set(unlisted))
|
||||
|
||||
|
||||
def _counted_channels(user: User):
|
||||
"""The channels that make the list by their counted videos, whatever the catalog holds of them."""
|
||||
return (
|
||||
select(TakeoutItem.channel_id)
|
||||
.where(_unfollowed(user), TakeoutItem.duration_seconds >= WATCHED_CHANNEL_MIN_SECONDS)
|
||||
.group_by(TakeoutItem.channel_id)
|
||||
.having(func.count() >= WATCHED_CHANNEL_MIN_VIDEOS)
|
||||
.subquery()
|
||||
)
|
||||
|
||||
|
||||
def watched_channels_to_describe(db: Session, user: User) -> list[str]:
|
||||
"""The channels on the list (by their counted videos) that the catalog lacks or has no details
|
||||
of yet: what the list cannot name, or names with no more than a video brought."""
|
||||
counted = _counted_channels(user)
|
||||
return list(
|
||||
db.execute(
|
||||
select(counted.c.channel_id)
|
||||
.outerjoin(Channel, Channel.id == counted.c.channel_id)
|
||||
.where(Channel.details_synced_at.is_(None))
|
||||
).scalars()
|
||||
)
|
||||
|
||||
|
||||
def watched_channels(db: Session, user: User) -> list[tuple[Channel, int, datetime]]:
|
||||
"""The channels the user watched often but does not follow: (channel, how many of its videos
|
||||
over five minutes they watched, when last). Most watched first; on a tie the one watched more
|
||||
lately, which is the likelier interest."""
|
||||
n = func.count(TakeoutItem.id)
|
||||
last = func.max(TakeoutItem.at)
|
||||
return list(
|
||||
db.execute(
|
||||
select(Channel, n, last)
|
||||
.join(Channel, Channel.id == TakeoutItem.channel_id)
|
||||
.where(_unfollowed(user), TakeoutItem.duration_seconds >= WATCHED_CHANNEL_MIN_SECONDS)
|
||||
.group_by(Channel.id)
|
||||
.having(n >= WATCHED_CHANNEL_MIN_VIDEOS)
|
||||
.order_by(n.desc(), last.desc(), Channel.id)
|
||||
).tuples()
|
||||
)
|
||||
|
||||
|
||||
def has_watch_history(db: Session, user: User) -> bool:
|
||||
"""Whether the user has uploaded a Takeout watch history at all."""
|
||||
return bool(
|
||||
db.scalar(
|
||||
select(
|
||||
select(TakeoutItem.id)
|
||||
.where(owned_by(TakeoutItem.user_id, user), TakeoutItem.kind == TakeoutKind.WATCHED)
|
||||
.exists()
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def watched_in_history(db: Session, user: User, channel_id: str) -> bool:
|
||||
"""Whether the user's Takeout history shows them watching this channel at all."""
|
||||
return bool(
|
||||
db.scalar(
|
||||
select(
|
||||
select(TakeoutItem.id)
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||
TakeoutItem.channel_id == channel_id,
|
||||
)
|
||||
.exists()
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def watched_channel_videos(
|
||||
db: Session, user: User, channel_id: str
|
||||
) -> tuple[list[tuple[str, datetime]], int]:
|
||||
"""The user's watched videos of one channel that the list counts, last watched first: (video
|
||||
id, when). And how many YouTube no longer has."""
|
||||
rows = db.execute(
|
||||
select(TakeoutItem.video_id, TakeoutItem.at, TakeoutItem.duration_seconds)
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
TakeoutItem.kind == TakeoutKind.WATCHED,
|
||||
TakeoutItem.channel_id == channel_id,
|
||||
TakeoutItem.looked_up_at.is_not(None),
|
||||
)
|
||||
.order_by(TakeoutItem.at.desc(), TakeoutItem.video_id)
|
||||
).tuples()
|
||||
counted: list[tuple[str, datetime]] = []
|
||||
gone = 0
|
||||
for vid, at, seconds in rows:
|
||||
if seconds is None:
|
||||
gone += 1
|
||||
elif seconds >= WATCHED_CHANNEL_MIN_SECONDS:
|
||||
counted.append((vid, at))
|
||||
return counted, gone
|
||||
|
||||
|
||||
def takeout_users_with_list_work(db: Session) -> list[int]:
|
||||
"""Every user whose "watched but not followed" list a run can still add to: lengths to ask, or
|
||||
channels to name. Asked of each user whose history holds a channel of enough watched videos,
|
||||
since what is off a list (followed, blocked, put out, their own) is that user's own — a row of
|
||||
one is never asked about, and would otherwise bring the user back on every run."""
|
||||
per_channel = (
|
||||
select(TakeoutItem.user_id)
|
||||
.where(TakeoutItem.kind == TakeoutKind.WATCHED, TakeoutItem.channel_id.is_not(None))
|
||||
.group_by(TakeoutItem.user_id, TakeoutItem.channel_id)
|
||||
.having(func.count() >= WATCHED_CHANNEL_MIN_VIDEOS)
|
||||
.subquery()
|
||||
)
|
||||
users = db.execute(
|
||||
select(User).where(User.id.in_(select(per_channel.c.user_id))).order_by(User.id)
|
||||
).scalars()
|
||||
return [user.id for user in users if watched_channels_pending(db, user)]
|
||||
|
||||
@@ -9,13 +9,27 @@ unique key, a row lock taken before a presence check. The comfort shelf's writes
|
||||
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import exists, func, select, text, update
|
||||
from typing import cast
|
||||
|
||||
from sqlalchemy import Table, bindparam, delete, exists, func, select, text, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import User
|
||||
from app.reader import queries
|
||||
from app.reader.models import ChannelTag, Playlist, PlaylistItem, SavedView, Tag, VideoState
|
||||
from app.reader.models import (
|
||||
ChannelTag,
|
||||
Playlist,
|
||||
PlaylistItem,
|
||||
SavedView,
|
||||
Tag,
|
||||
TakeoutChannelDismissal,
|
||||
TakeoutItem,
|
||||
TakeoutKind,
|
||||
TakeoutOutcome,
|
||||
VideoState,
|
||||
VideoUnmark,
|
||||
)
|
||||
from app.sync.playlists import lock_playlists, recompute_dirty
|
||||
from app.userscope import owned_by
|
||||
|
||||
@@ -196,6 +210,7 @@ def unmark(db: Session, user: User, video_id: str) -> None:
|
||||
the in-progress state, not where the user left off); otherwise the row goes. The player fires
|
||||
this beside a checkpoint that may delete the same row: the caller's commit then raises
|
||||
`StaleDataError`, which is that lost race and nothing worse."""
|
||||
lock_takeout(db, user) # an import settling meanwhile reads the un-mark below, or comes first
|
||||
row = queries.video_state(db, user, video_id)
|
||||
if row is None:
|
||||
return
|
||||
@@ -204,9 +219,21 @@ def unmark(db: Session, user: User, video_id: str) -> None:
|
||||
row.watched_at = None
|
||||
else:
|
||||
db.delete(row)
|
||||
_remember_unmark(db, user, video_id)
|
||||
db.flush()
|
||||
|
||||
|
||||
def _remember_unmark(db: Session, user: User, video_id: str) -> None:
|
||||
"""Note that the user took the video back to "new" now: the state they set leaves no row, and
|
||||
an import of their YouTube history must not mark it watched again (`apply_takeout_watched`)."""
|
||||
now = datetime.now(timezone.utc)
|
||||
db.execute(
|
||||
pg_insert(VideoUnmark)
|
||||
.values(user_id=user.id, video_id=video_id, at=now)
|
||||
.on_conflict_do_update(constraint="uq_video_unmark", set_={"at": now})
|
||||
)
|
||||
|
||||
|
||||
def mark(db: Session, user: User, video_id: str, status: str) -> None:
|
||||
""" "watched" or "hidden", as one atomic upsert on (user, video). The player marks "watched" at
|
||||
the end of playback at the instant a checkpoint may delete the in-progress row; a SELECT then
|
||||
@@ -231,9 +258,11 @@ def mark(db: Session, user: User, video_id: str, status: str) -> None:
|
||||
def clear_state(db: Session, user: User, video_id: str) -> None:
|
||||
"""The video back to pristine for this user: status, resume point and `watched_at` all go.
|
||||
Playlist membership lives elsewhere and stays."""
|
||||
lock_takeout(db, user) # as `unmark`
|
||||
row = queries.video_state(db, user, video_id)
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
_remember_unmark(db, user, video_id)
|
||||
db.flush()
|
||||
|
||||
|
||||
@@ -495,3 +524,192 @@ def detach_user_tag(db: Session, user: User, channel_id: str, tag_id: int) -> No
|
||||
if row is not None:
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
|
||||
|
||||
# --- Google Takeout --------------------------------------------------------------------------
|
||||
|
||||
_TAKEOUT_BATCH = 1000
|
||||
# The advisory-lock namespace of one user's Takeout rows ("TO"), so a key cannot meet another lock's.
|
||||
_TAKEOUT_LOCK_NS = 0x544F
|
||||
|
||||
|
||||
def lock_takeout(db: Session, user: User) -> None:
|
||||
"""Hold the user's Takeout rows for the rest of the transaction. An upload's staging and an
|
||||
import's settling each lock many of the same rows, one statement after another and in no common
|
||||
order, so without one lock over both the two can deadlock. An un-mark takes it too: the import
|
||||
reads the un-marks before it marks, and one landing between the two would be overwritten."""
|
||||
db.execute(select(func.pg_advisory_xact_lock(_TAKEOUT_LOCK_NS, user.id)))
|
||||
|
||||
|
||||
def stage_takeout(
|
||||
db: Session, user: User, kind: TakeoutKind, rows: dict[str, tuple[datetime, str | None]]
|
||||
) -> None:
|
||||
"""Keep the videos of one Takeout file, `{video id: (time, channel id)}`, for the import job.
|
||||
A second upload of the same export, or a newer one, adds what is new: a video already kept keeps
|
||||
the later time, and a row the import has settled keeps its outcome."""
|
||||
lock_takeout(db, user)
|
||||
values = [
|
||||
{"user_id": user.id, "kind": kind, "video_id": vid, "at": at, "channel_id": channel}
|
||||
for vid, (at, channel) in rows.items()
|
||||
]
|
||||
for start in range(0, len(values), _TAKEOUT_BATCH):
|
||||
stmt = pg_insert(TakeoutItem).values(values[start : start + _TAKEOUT_BATCH])
|
||||
db.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
constraint="uq_takeout_item",
|
||||
set_={
|
||||
"at": func.greatest(TakeoutItem.at, stmt.excluded.at),
|
||||
"channel_id": func.coalesce(stmt.excluded.channel_id, TakeoutItem.channel_id),
|
||||
},
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def _settle(db: Session, ids: list[int], outcome: TakeoutOutcome) -> None:
|
||||
for start in range(0, len(ids), _TAKEOUT_BATCH):
|
||||
db.execute(
|
||||
update(TakeoutItem)
|
||||
.where(TakeoutItem.id.in_(ids[start : start + _TAKEOUT_BATCH]))
|
||||
.values(outcome=outcome)
|
||||
)
|
||||
|
||||
|
||||
def apply_takeout_watched(db: Session, user: User) -> dict[str, int]:
|
||||
"""Mark watched what the user watched on YouTube, for the videos the catalog knows and has probed
|
||||
for Shorts; the rest wait. `watched_at` is the Takeout time. A Short is not marked (swiping one
|
||||
past is not watching it), and a state the user set here wins: watched or hidden, anything
|
||||
changed after the Takeout time, or an un-mark since then (`VideoUnmark`). The upsert's own
|
||||
condition is that rule, so a state set while the import runs wins too."""
|
||||
lock_takeout(db, user)
|
||||
short: list[int] = []
|
||||
todo: dict[str, tuple[int, datetime]] = {}
|
||||
ready = queries.takeout_watched_ready(db, user)
|
||||
unmarked = queries.unmarked_at(db, user, [video_id for _, video_id, _, _ in ready])
|
||||
kept: list[int] = []
|
||||
for row_id, video_id, at, is_short in ready:
|
||||
if is_short:
|
||||
short.append(row_id)
|
||||
elif video_id in unmarked and unmarked[video_id] >= at:
|
||||
# Un-marked here since they watched it there: theirs, as a state they set is.
|
||||
kept.append(row_id)
|
||||
else:
|
||||
todo[video_id] = (row_id, at)
|
||||
marked: set[str] = set()
|
||||
items = list(todo.items())
|
||||
now = datetime.now(timezone.utc)
|
||||
for start in range(0, len(items), _TAKEOUT_BATCH):
|
||||
stmt = pg_insert(VideoState).values(
|
||||
[
|
||||
{"user_id": user.id, "video_id": vid, "status": "watched", "watched_at": at}
|
||||
for vid, (_, at) in items[start : start + _TAKEOUT_BATCH]
|
||||
]
|
||||
)
|
||||
marked.update(
|
||||
db.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
constraint="uq_user_video",
|
||||
set_={
|
||||
"status": "watched",
|
||||
"watched_at": stmt.excluded.watched_at,
|
||||
"updated_at": now,
|
||||
},
|
||||
where=(VideoState.status == "new")
|
||||
& (VideoState.updated_at < stmt.excluded.watched_at),
|
||||
).returning(VideoState.video_id)
|
||||
).scalars()
|
||||
)
|
||||
kept += [row_id for vid, (row_id, _) in items if vid not in marked]
|
||||
_settle(db, [todo[vid][0] for vid in marked], TakeoutOutcome.WATCHED)
|
||||
_settle(db, kept, TakeoutOutcome.KEPT)
|
||||
_settle(db, short, TakeoutOutcome.SHORT)
|
||||
db.flush()
|
||||
return {"watched": len(marked), "kept": len(kept), "shorts": len(short)}
|
||||
|
||||
|
||||
def apply_takeout_watch_later(
|
||||
db: Session, user: User, rows: list[tuple[int, str]], asked: set[str]
|
||||
) -> dict[str, int]:
|
||||
"""Put the videos of the user's waiting Watch later `rows`, (row id, video id) oldest first, into
|
||||
their list here, after what it already holds. `asked` is the videos YouTube was just asked for: one
|
||||
of those the catalog still lacks is gone from YouTube; any other unknown one keeps waiting."""
|
||||
if not rows:
|
||||
return {}
|
||||
lock_takeout(db, user)
|
||||
ids = [vid for _, vid in rows]
|
||||
known = queries.known_video_ids(db, ids)
|
||||
pl = watch_later(db, user)
|
||||
# What add_item does a video at a time, for the whole list at once: one presence read and one
|
||||
# insert, not four statements a video (a full list is thousands). Locked before the read, as there.
|
||||
next_position = next_item_position(db, pl.id)
|
||||
held = queries.in_playlist(db, pl.id, [vid for vid in ids if vid in known])
|
||||
done: dict[TakeoutOutcome, list[int]] = {o: [] for o in TakeoutOutcome}
|
||||
for row_id, vid in rows:
|
||||
if vid not in known:
|
||||
if vid in asked:
|
||||
done[TakeoutOutcome.MISSING].append(row_id)
|
||||
elif vid in held:
|
||||
done[TakeoutOutcome.PRESENT].append(row_id)
|
||||
else:
|
||||
db.add(PlaylistItem(playlist_id=pl.id, video_id=vid, position=next_position))
|
||||
next_position += 1
|
||||
held.add(vid)
|
||||
done[TakeoutOutcome.ADDED].append(row_id)
|
||||
if done[TakeoutOutcome.ADDED]:
|
||||
touch(pl)
|
||||
for outcome, ids in done.items():
|
||||
_settle(db, ids, outcome)
|
||||
db.flush()
|
||||
return {
|
||||
"watch_later_added": len(done[TakeoutOutcome.ADDED]),
|
||||
"watch_later_present": len(done[TakeoutOutcome.PRESENT]),
|
||||
"watch_later_missing": len(done[TakeoutOutcome.MISSING]),
|
||||
}
|
||||
|
||||
|
||||
# --- channels watched but not followed ---------------------------------------------------------
|
||||
|
||||
|
||||
def record_takeout_lengths(
|
||||
db: Session, user: User, asked: list[str], lengths: dict[str, int | None]
|
||||
) -> None:
|
||||
"""Keep what YouTube said of the user's watched videos `asked` about: each one's length, or none
|
||||
for a video it no longer has (`lengths` leaves it out). Every asked row is marked asked, so a
|
||||
video YouTube lost is not asked about again."""
|
||||
if not asked:
|
||||
return
|
||||
lock_takeout(db, user) # these rows are an upload's to update too
|
||||
now = datetime.now(timezone.utc)
|
||||
table = cast(Table, TakeoutItem.__table__)
|
||||
db.execute(
|
||||
table.update()
|
||||
.where(
|
||||
owned_by(TakeoutItem.user_id, user),
|
||||
table.c.kind == TakeoutKind.WATCHED,
|
||||
table.c.video_id == bindparam("vid"),
|
||||
)
|
||||
.values(looked_up_at=now, duration_seconds=bindparam("seconds")),
|
||||
[{"vid": vid, "seconds": lengths.get(vid)} for vid in asked],
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def dismiss_watched_channel(db: Session, user: User, channel_id: str) -> None:
|
||||
"""Put a channel out of the user's "watched but not followed" list (again is a no-op)."""
|
||||
db.execute(
|
||||
pg_insert(TakeoutChannelDismissal)
|
||||
.values(user_id=user.id, channel_id=channel_id)
|
||||
.on_conflict_do_nothing(constraint="uq_takeout_channel_dismissal")
|
||||
)
|
||||
db.flush()
|
||||
|
||||
|
||||
def restore_watched_channel(db: Session, user: User, channel_id: str) -> None:
|
||||
"""Let a channel the user put out of the list back on it."""
|
||||
db.execute(
|
||||
delete(TakeoutChannelDismissal).where(
|
||||
owned_by(TakeoutChannelDismissal.user_id, user),
|
||||
TakeoutChannelDismissal.channel_id == channel_id,
|
||||
)
|
||||
)
|
||||
db.flush()
|
||||
|
||||
@@ -0,0 +1,497 @@
|
||||
"""A Google Takeout export, imported: what the user watched on YouTube becomes watched here, and their
|
||||
YouTube Watch later list joins theirs. The YouTube API reads neither of them, so the export is the
|
||||
only way in.
|
||||
|
||||
Two files of the export are read, each uploaded on its own: `history/watch-history.json` and
|
||||
`playlists/Watch later-videos.csv`. Their videos are kept (`TakeoutItem`) and the import job applies
|
||||
what it can; a video the catalog does not know yet waits for it. The watched videos' channels also
|
||||
make the list of channels the user watches but does not follow (`queries.watched_channels`): the
|
||||
job asks YouTube how long those videos are, and what the channels are, for it.
|
||||
|
||||
The format, measured on a real export (2026-10): the history is a JSON list, one entry per view,
|
||||
newest first. A view of a video has a `titleUrl` with `watch?v=<id>` (on www.youtube.com or
|
||||
music.youtube.com) and a `time`, with or without milliseconds. The channel is the first `subtitles`
|
||||
entry's url, absent for a video since removed. An ad is an entry whose `details` say "From Google
|
||||
Ads". The title is in the account's language, so nothing here reads it. A Short is a plain
|
||||
`watch?v=` view, told apart only once the catalog has probed the video. The Watch later csv is a
|
||||
header row (in the account's language) and `<id>,<ISO time added>` rows.
|
||||
"""
|
||||
|
||||
import codecs
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, quota
|
||||
from app.bodylimit import TAKEOUT_MAX_BYTES
|
||||
from app.db import SessionLocal
|
||||
from app.models import User
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.reader import queries, service
|
||||
from app.reader.models import TakeoutKind
|
||||
from app.sync.runner import get_service_user
|
||||
from app.sync.subscriptions import describe_channels
|
||||
from app.sync.videos import ensure_videos, parse_iso8601_duration
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
|
||||
log = logs.get_logger(__name__)
|
||||
|
||||
# The largest file the upload takes (its edge cap and the route's own are one number).
|
||||
MAX_BYTES = TAKEOUT_MAX_BYTES
|
||||
# The most videos a Watch later file may hold: YouTube's own cap on one playlist. Each unknown one is
|
||||
# looked up on the shared API budget, so a longer list is not one YouTube made.
|
||||
WATCH_LATER_MAX = 5000
|
||||
# The most Takeout videos one user keeps: twenty times the real export measured here (~9,000 videos
|
||||
# across both files), so it only stops uploads of made-up ones piling up for good.
|
||||
ROWS_MAX = 200_000
|
||||
|
||||
_VIDEO_ID = re.compile(r"^[A-Za-z0-9_-]{11}$")
|
||||
_WATCH_URL = re.compile(
|
||||
r"^https?://(?:www\.|music\.|m\.)?youtube\.com/watch\?v=([A-Za-z0-9_-]{11})"
|
||||
)
|
||||
_CHANNEL_URL = re.compile(r"/channel/(UC[A-Za-z0-9_-]{22})")
|
||||
|
||||
|
||||
class TakeoutError(ValueError):
|
||||
"""The file is not one the import reads; `reason` is the client's word for why."""
|
||||
|
||||
def __init__(self, reason: str):
|
||||
super().__init__(reason)
|
||||
self.reason = reason
|
||||
|
||||
|
||||
@dataclass
|
||||
class Parsed:
|
||||
"""One file's videos, `{video id: (time, channel id)}`, and what it left out, counted."""
|
||||
|
||||
kind: TakeoutKind
|
||||
videos: dict[str, tuple[datetime, str | None]] = field(default_factory=dict)
|
||||
entries: int = 0
|
||||
ads: int = 0
|
||||
other: int = 0 # not a view of a video (a post, a story) or a video with no link left
|
||||
malformed: int = 0
|
||||
|
||||
|
||||
def _time(raw: object) -> datetime | None:
|
||||
if not isinstance(raw, str):
|
||||
return None
|
||||
try:
|
||||
at = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
# silent-ok: the row is counted as malformed, and the count is in the upload's answer.
|
||||
except ValueError:
|
||||
return None
|
||||
return at if at.tzinfo is not None else None
|
||||
|
||||
|
||||
def _keep(parsed: Parsed, vid: str, at: datetime, channel: str | None) -> None:
|
||||
old = parsed.videos.get(vid)
|
||||
if old is None or at > old[0]:
|
||||
parsed.videos[vid] = (at, channel or (old[1] if old else None))
|
||||
|
||||
|
||||
def _history(data: Iterator[object]) -> Parsed:
|
||||
parsed = Parsed(TakeoutKind.WATCHED)
|
||||
for entry in data:
|
||||
parsed.entries += 1
|
||||
if not isinstance(entry, dict):
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
details = entry.get("details")
|
||||
if isinstance(details, list) and any(
|
||||
isinstance(d, dict) and d.get("name") == "From Google Ads" for d in details
|
||||
):
|
||||
parsed.ads += 1
|
||||
continue
|
||||
url = entry.get("titleUrl")
|
||||
match = _WATCH_URL.match(url) if isinstance(url, str) else None
|
||||
if match is None:
|
||||
parsed.other += 1
|
||||
continue
|
||||
at = _time(entry.get("time"))
|
||||
if at is None:
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
channel = None
|
||||
subtitles = entry.get("subtitles")
|
||||
if isinstance(subtitles, list) and subtitles and isinstance(subtitles[0], dict):
|
||||
found = _CHANNEL_URL.search(str(subtitles[0].get("url", "")))
|
||||
channel = found.group(1) if found else None
|
||||
_keep(parsed, match.group(1), at, channel)
|
||||
return parsed
|
||||
|
||||
|
||||
def _watch_later(text: str) -> Parsed:
|
||||
parsed = Parsed(TakeoutKind.WATCH_LATER)
|
||||
rows = (row for row in csv.reader(io.StringIO(text)) if any(cell.strip() for cell in row))
|
||||
first = next(rows, None)
|
||||
if first is None or len(first) < 2:
|
||||
raise TakeoutError("not_takeout")
|
||||
# The first row is the header, in the account's language, unless it is already a video.
|
||||
for row in chain([first], rows) if _VIDEO_ID.match(first[0].strip()) else rows:
|
||||
parsed.entries += 1
|
||||
vid = row[0].strip()
|
||||
at = _time(row[1].strip()) if len(row) > 1 else None
|
||||
if not _VIDEO_ID.match(vid) or at is None:
|
||||
parsed.malformed += 1
|
||||
continue
|
||||
_keep(parsed, vid, at, None)
|
||||
if len(parsed.videos) > WATCH_LATER_MAX:
|
||||
raise TakeoutError("too_many")
|
||||
return parsed
|
||||
|
||||
|
||||
def _skip(text: str, at: int) -> int:
|
||||
"""The first index at or after `at` that is not JSON whitespace."""
|
||||
while at < len(text) and text[at] in " \t\n\r":
|
||||
at += 1
|
||||
return at
|
||||
|
||||
|
||||
# How much of the file is decoded at a time. Decoded whole, the text costs up to four bytes a
|
||||
# character (one emoji in a title is enough), several times the file on top of its bytes.
|
||||
_WINDOW = 1024 * 1024
|
||||
|
||||
|
||||
class _Window:
|
||||
"""The file's text, decoded a window at a time: `text` holds what is not read yet from `at` on.
|
||||
The bytes are the caller's, so only the window is this reader's memory."""
|
||||
|
||||
def __init__(self, body: bytes, start: int):
|
||||
self._body, self._next = body, start
|
||||
self._decoder = codecs.getincrementaldecoder("utf-8")() # strict: bad UTF-8 raises
|
||||
self.text, self.at = "", 0
|
||||
|
||||
@property
|
||||
def done(self) -> bool:
|
||||
return self._next >= len(self._body)
|
||||
|
||||
def more(self, at_least: int = 0) -> None:
|
||||
"""Decode the next window (at least `at_least` bytes), dropping what is read already."""
|
||||
self.text, self.at = self.text[self.at :], 0
|
||||
size = max(_WINDOW, at_least)
|
||||
chunk = self._body[self._next : self._next + size]
|
||||
self._next += len(chunk)
|
||||
self.text += self._decoder.decode(chunk, final=self.done)
|
||||
|
||||
def skip(self) -> None:
|
||||
"""Past the whitespace, reading on as far as it goes."""
|
||||
while True:
|
||||
self.at = _skip(self.text, self.at)
|
||||
if self.at < len(self.text) or self.done:
|
||||
return
|
||||
self.more()
|
||||
|
||||
def startswith(self, char: str) -> bool:
|
||||
self.skip()
|
||||
return self.text.startswith(char, self.at)
|
||||
|
||||
|
||||
def _decode(decoder: json.JSONDecoder, text: str, at: int) -> tuple[object, int] | None:
|
||||
"""The JSON value at `at` and where it ends, or None where none reads (yet)."""
|
||||
try:
|
||||
return decoder.raw_decode(text, at)
|
||||
# silent-ok: not a failure here — the caller reads on, or refuses the file once it has all.
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _json_list(body: bytes, start: int) -> Iterator[object]:
|
||||
"""The entries of the JSON list at byte `start` (its "["), one at a time: the whole tree is never
|
||||
built, nor the whole text, so a file of a million tiny entries costs a window and one entry, not
|
||||
a million dicts at once."""
|
||||
decoder = json.JSONDecoder()
|
||||
window = _Window(body, start + 1) # past the "[" the caller saw
|
||||
if window.startswith("]"):
|
||||
window.at += 1
|
||||
else:
|
||||
while True:
|
||||
window.skip()
|
||||
read = _decode(decoder, window.text, window.at)
|
||||
# An entry cut by the window's edge fails, or (a number) reads short: read on and try
|
||||
# again, with a window twice what is left, so one entry larger than any window is read
|
||||
# in a few tries, not one a window.
|
||||
if read is None or (read[1] == len(window.text) and not window.done):
|
||||
if window.done:
|
||||
raise ValueError("not a JSON list")
|
||||
window.more(2 * (len(window.text) - window.at))
|
||||
continue
|
||||
entry, window.at = read
|
||||
yield entry
|
||||
if window.startswith(","):
|
||||
window.at += 1
|
||||
elif window.startswith("]"):
|
||||
window.at += 1
|
||||
break
|
||||
else:
|
||||
raise ValueError("not a JSON list")
|
||||
window.skip()
|
||||
if window.at != len(window.text):
|
||||
raise ValueError("text after the JSON list")
|
||||
|
||||
|
||||
# The largest Watch later file read: YouTube's longest list is some 200 KB of rows, so a file ten
|
||||
# times that is not one — and, unlike the history, it is decoded whole.
|
||||
CSV_MAX_BYTES = 2 * 1024 * 1024
|
||||
_BOM = codecs.BOM_UTF8
|
||||
|
||||
|
||||
def parse(body: bytes) -> Parsed:
|
||||
"""Read an uploaded Takeout file: the watch history (JSON) or the Watch later list (csv), told
|
||||
apart by their content, not their name. Raises `TakeoutError` for anything else, and for a file
|
||||
with nothing to import in it."""
|
||||
start = len(_BOM) if body.startswith(_BOM) else 0
|
||||
while start < len(body) and body[start] in b" \t\n\r\f\v":
|
||||
start += 1
|
||||
first = body[start : start + 1]
|
||||
if first in (b"{", b"<"):
|
||||
# Another JSON file of the export, or the history's HTML variant.
|
||||
raise TakeoutError("not_takeout")
|
||||
if first != b"[" and len(body) - start > CSV_MAX_BYTES:
|
||||
raise TakeoutError("too_many")
|
||||
try:
|
||||
# A bad UTF-8 sequence is a ValueError too (UnicodeDecodeError), read below as not a file.
|
||||
parsed = (
|
||||
_history(_json_list(body, start))
|
||||
if first == b"["
|
||||
else _watch_later(body[start:].decode("utf-8"))
|
||||
)
|
||||
# A file the import cannot read, however it fails: bad JSON (a ValueError, as is a number too
|
||||
# long to convert), nesting deeper than the decoder recurses, a csv field past csv's limit.
|
||||
except (ValueError, RecursionError, csv.Error) as exc:
|
||||
if isinstance(exc, TakeoutError):
|
||||
raise
|
||||
raise TakeoutError("not_takeout") from None
|
||||
if not parsed.videos:
|
||||
raise TakeoutError("no_videos")
|
||||
return parsed
|
||||
|
||||
|
||||
# One import per user at a time: a second file uploaded while the first one's run is under way runs
|
||||
# after it (the scheduler's ad-hoc queue), but the Shorts job settles the same users' rows on a
|
||||
# thread of its own, and the two would settle the same rows. One API process runs the scheduler, so a
|
||||
# lock in this module holds it (as the channel backfill's does).
|
||||
_user_locks: dict[int, threading.Lock] = {}
|
||||
_user_locks_guard = threading.Lock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _one_import(user_id: int) -> Iterator[None]:
|
||||
with _user_locks_guard:
|
||||
lock = _user_locks.setdefault(user_id, threading.Lock())
|
||||
with lock:
|
||||
yield
|
||||
|
||||
|
||||
def run_target(user_id: int) -> str:
|
||||
"""The import run's target, `job_runs.target`: one user's import."""
|
||||
return f"user:{user_id}"
|
||||
|
||||
|
||||
def target_title(db: Session, target: str) -> str | None:
|
||||
"""Who an import run's target is, for the admin's run lists: the user's name, else their email."""
|
||||
_, _, raw = target.partition("user:")
|
||||
user = db.get(User, int(raw)) if raw.isdigit() else None
|
||||
return (user.display_name or user.email) if user is not None else None
|
||||
|
||||
|
||||
class TooMany(Exception):
|
||||
"""The upload would take the user past `ROWS_MAX` kept videos."""
|
||||
|
||||
|
||||
def stage(db: Session, user: User, parsed: Parsed) -> None:
|
||||
"""Keep an uploaded file's videos for the import, unless that would take the user past
|
||||
`ROWS_MAX` (`TooMany`). The caller commits."""
|
||||
service.lock_takeout(db, user) # the count and the insert are one user's, not two uploads' each
|
||||
if (
|
||||
queries.takeout_rows_besides(db, user, parsed.kind, parsed.videos) + len(parsed.videos)
|
||||
> ROWS_MAX
|
||||
):
|
||||
raise TooMany
|
||||
service.stage_takeout(db, user, parsed.kind, parsed.videos)
|
||||
|
||||
|
||||
# How many unknown Watch later videos one user may have looked up on the shared API budget in a day:
|
||||
# two full Watch later lists (two units a hundred ids). The rest wait for the next day's runs. Both
|
||||
# shares pace the work, they do not bound its cost: they are held in memory, so a restart starts
|
||||
# them afresh, which only brings the rest forward — every video is asked about once (a looked-up
|
||||
# row is marked), and `quota.backfill_reserve_reached` still guards the shared budget.
|
||||
_lookups = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600)
|
||||
# How many watched videos' lengths one user may ask about in a day for the "watched but not followed"
|
||||
# list (a unit a fifty). A share of its own: a large history asks for thousands, and on the Watch
|
||||
# later share it would leave the export's Watch later file waiting a day behind it.
|
||||
_lengths = RateLimiter(max_events=2 * WATCH_LATER_MAX, window_seconds=24 * 3600)
|
||||
|
||||
|
||||
def _put_in_watch_later(db: Session, user: User) -> dict:
|
||||
"""Put the user's waiting Watch later videos into theirs here: the ones the catalog knows at once,
|
||||
the others once YouTube was asked for them — when a YouTube-connected account can ask, today's
|
||||
budget is above the backfill reserve, and the user's own daily share of lookups allows it. A video
|
||||
not asked about yet waits for a later run. The lookup commits what it fetched; the caller commits
|
||||
the rest."""
|
||||
waiting = queries.takeout_waiting(db, user, TakeoutKind.WATCH_LATER)
|
||||
if not waiting:
|
||||
return {}
|
||||
ids = [vid for _, vid in waiting]
|
||||
known = queries.known_video_ids(db, ids)
|
||||
ask = [vid for vid in ids if vid not in known][:WATCH_LATER_MAX]
|
||||
asked: set[str] = set()
|
||||
fetcher = get_service_user(db) if ask else None
|
||||
if (
|
||||
fetcher is not None
|
||||
and not quota.backfill_reserve_reached()
|
||||
and _lookups.room(str(user.id)) >= len(ask)
|
||||
):
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP):
|
||||
with YouTubeClient(db, fetcher) as yt:
|
||||
ensure_videos(db, yt, ask)
|
||||
# Counted once done: a lookup that failed spent none of the share (one run per user at a
|
||||
# time, so the room asked above is still there).
|
||||
_lookups.allow(str(user.id), cost=len(ask))
|
||||
asked = set(ask)
|
||||
# Only the rows read above: one a second upload staged meanwhile was never asked about.
|
||||
return service.apply_takeout_watch_later(db, user, waiting, asked)
|
||||
|
||||
|
||||
def _ask_lengths(db: Session, user: User) -> dict:
|
||||
"""Ask YouTube how long the user's watched videos are that the "watched but not followed" list
|
||||
still needs, and keep the answer — on the same terms as the Watch later lookups, on a daily
|
||||
share of its own. What is not asked yet (no YouTube-connected account, the reserve reached, the
|
||||
share spent) waits for a later run, and the list says it is not complete. A YouTube failure is
|
||||
logged and left for a later run too: it is no reason to fail the import it follows. The caller
|
||||
commits."""
|
||||
ask = queries.takeout_lengths_to_ask(db, user)
|
||||
fetcher = get_service_user(db) if ask else None
|
||||
if fetcher is None or quota.backfill_reserve_reached():
|
||||
return {}
|
||||
ask = ask[: int(_lengths.room(str(user.id)))]
|
||||
if not ask:
|
||||
return {}
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP):
|
||||
with YouTubeClient(db, fetcher) as yt:
|
||||
items = yt.get_videos(ask)
|
||||
except YouTubeError as exc:
|
||||
log.warning(
|
||||
"Takeout: asking the watched videos' lengths failed: %s",
|
||||
exc,
|
||||
extra=logs.event("takeout.lengths.failed", user_id=user.id, reason=exc.reason),
|
||||
)
|
||||
return {}
|
||||
_lengths.allow(str(user.id), cost=len(ask))
|
||||
lengths = {
|
||||
it["id"]: parse_iso8601_duration(it.get("contentDetails", {}).get("duration"))
|
||||
for it in items
|
||||
if it.get("id")
|
||||
}
|
||||
service.record_takeout_lengths(db, user, ask, lengths)
|
||||
return {"lengths_asked": len(ask)}
|
||||
|
||||
|
||||
def _describe_channels(db: Session, user: User) -> dict:
|
||||
"""Bring the channels the user's list holds into the catalog with their details (name, picture,
|
||||
description), the ones it lacks: without them the list cannot show a channel. A YouTube failure
|
||||
is logged and left for a later run. The caller commits."""
|
||||
need = queries.watched_channels_to_describe(db, user)
|
||||
fetcher = get_service_user(db) if need else None
|
||||
if fetcher is None or quota.backfill_reserve_reached():
|
||||
return {}
|
||||
try:
|
||||
with quota.attribute(user.id, quota.QuotaAction.CHANNELS_DISCOVER):
|
||||
with YouTubeClient(db, fetcher) as yt:
|
||||
describe_channels(db, yt, need)
|
||||
except YouTubeError as exc:
|
||||
db.rollback()
|
||||
log.warning(
|
||||
"Takeout: describing the watched channels failed: %s",
|
||||
exc,
|
||||
extra=logs.event("takeout.channels.failed", user_id=user.id, reason=exc.reason),
|
||||
)
|
||||
return {}
|
||||
return {"channels_described": len(need)}
|
||||
|
||||
|
||||
def _channel_list(db: Session, user: User) -> dict:
|
||||
"""The "watched but not followed" list's part of a run: the lengths, landed, then the channels.
|
||||
Commits each."""
|
||||
result = _ask_lengths(db, user)
|
||||
db.commit()
|
||||
result |= _describe_channels(db, user)
|
||||
db.commit()
|
||||
return result
|
||||
|
||||
|
||||
def run_import(user_id: int) -> dict:
|
||||
"""The import job: apply what the user's kept Takeout rows can say now, as one recorded run.
|
||||
The watched part commits before the Watch later part asks YouTube for the videos the catalog
|
||||
lacks, so a YouTube failure there loses nothing already marked; the "watched but not followed"
|
||||
list's lookups come last. Returns the counts."""
|
||||
with _one_import(user_id), SessionLocal() as db:
|
||||
user = db.get(User, user_id)
|
||||
if user is None:
|
||||
return {"skipped": "no user"}
|
||||
result = service.apply_takeout_watched(db, user)
|
||||
db.commit()
|
||||
result |= _put_in_watch_later(db, user)
|
||||
db.commit()
|
||||
result |= _channel_list(db, user)
|
||||
outcomes = queries.takeout_outcomes(db, user)
|
||||
result["waiting"] = outcomes.get((TakeoutKind.WATCHED, None), 0)
|
||||
if outcomes.get((TakeoutKind.WATCH_LATER, None)):
|
||||
result["watch_later_waiting"] = outcomes[(TakeoutKind.WATCH_LATER, None)]
|
||||
return result
|
||||
|
||||
|
||||
def apply_waiting(db: Session) -> dict:
|
||||
"""Settle what every user's import left waiting and can be settled now, one commit each: a
|
||||
watched video the catalog has come to know and the Shorts probe has cleared (by its job or a
|
||||
search's), the Watch later videos a run could not put in yet, and what the "watched but not
|
||||
followed" list still needs asked of YouTube. Run after each run of the
|
||||
`shorts` job, which is what clears the most of them. Returns its counts, or an empty dict when
|
||||
nothing was waiting that could move."""
|
||||
marked = added = asked = failed = 0
|
||||
ready = set(queries.takeout_users_ready(db))
|
||||
later = set(queries.takeout_users_watch_later_waiting(db))
|
||||
lengths = set(queries.takeout_users_with_list_work(db))
|
||||
for user_id in sorted(ready | later | lengths):
|
||||
user = db.get(User, user_id)
|
||||
if user is None:
|
||||
continue
|
||||
# One user's failure (a lookup YouTube refused) is theirs: it neither fails the job that
|
||||
# runs this nor keeps the users after them waiting. What it did not settle waits.
|
||||
try:
|
||||
with _one_import(user_id):
|
||||
if user_id in ready:
|
||||
marked += service.apply_takeout_watched(db, user)["watched"]
|
||||
db.commit()
|
||||
if user_id in later:
|
||||
added += _put_in_watch_later(db, user).get("watch_later_added", 0)
|
||||
db.commit()
|
||||
if user_id in lengths:
|
||||
asked += _channel_list(db, user).get("lengths_asked", 0)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
failed += 1
|
||||
log.exception(
|
||||
"Takeout settling failed for user %s",
|
||||
user_id,
|
||||
extra=logs.event("takeout.settle.failed", user_id=user_id),
|
||||
)
|
||||
out = {}
|
||||
if ready:
|
||||
out["takeout_watched"] = marked
|
||||
if added:
|
||||
out["takeout_watch_later_added"] = added
|
||||
if asked:
|
||||
out["takeout_lengths_asked"] = asked
|
||||
if failed:
|
||||
out["takeout_failed"] = failed
|
||||
return out
|
||||
@@ -124,6 +124,14 @@ async def capped_body(request: Request, limit: int) -> bytes:
|
||||
return bytes(body)
|
||||
|
||||
|
||||
async def drain_body(request: Request) -> None:
|
||||
"""Read the rest of the body to nowhere, before a refusal is raised: answered mid-upload, the
|
||||
client reports a network failure, not the refusal (`capped_stream_to` says why). `bodylimit`
|
||||
bounds how much there can be."""
|
||||
async for _ in request.stream():
|
||||
pass
|
||||
|
||||
|
||||
class ByteSink(Protocol):
|
||||
"""Where `capped_stream_to` writes: a file, or anything that takes the bytes on their way to one."""
|
||||
|
||||
|
||||
@@ -307,9 +307,9 @@ def explore_channel(
|
||||
"""Open an un-subscribed channel for browsing. Records the exploration (a per-user,
|
||||
grace-clocked row that gates this user's access to the channel's ephemeral videos and
|
||||
drives the later cleanup), flags the channel ephemeral unless someone organically follows
|
||||
it, then ingests one page of its uploads (newest-first) marked via_explore + enriched.
|
||||
Returns the next page token so the page can "load more". Guarded against the shared
|
||||
backfill quota reserve."""
|
||||
it, then ingests one page of its uploads (newest-first), enriched — marked via_explore while
|
||||
nobody follows it. Returns the next page token so the page can "load more". Guarded against
|
||||
the shared backfill quota reserve."""
|
||||
channel = _channel_or_404(db, channel_id)
|
||||
if queries.is_blocked(db, user, channel_id):
|
||||
raise refused(403, "channel_blocked")
|
||||
|
||||
+229
-6
@@ -1,19 +1,31 @@
|
||||
import threading
|
||||
from collections.abc import AsyncIterator
|
||||
from datetime import date
|
||||
from functools import partial
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from fastapi import APIRouter, Depends, Path, Query, Request
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.orm.exc import StaleDataError
|
||||
|
||||
from app import quota
|
||||
from app import jobruns, quota, scheduler
|
||||
from app.catalog import queries as catalog_queries, service as catalog_service
|
||||
from app.auth import current_user
|
||||
from app.db import get_db
|
||||
from app.db import get_db, release_read_connection
|
||||
from app.mailtext import lang_for_user
|
||||
from app.ratelimit import RateLimiter
|
||||
from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years
|
||||
from app.models import User
|
||||
from app.reader import queries, service
|
||||
from app.routes._common import ProgressIn, kept_position, not_found, youtube_refusal
|
||||
from app.sync.videos import parse_iso8601_duration
|
||||
from app.reader import queries, service, takeout
|
||||
from app.routes._common import (
|
||||
ProgressIn,
|
||||
capped_body,
|
||||
drain_body,
|
||||
kept_position,
|
||||
not_found,
|
||||
youtube_refusal,
|
||||
)
|
||||
from app.sync.videos import ensure_videos, parse_iso8601_duration
|
||||
from app.utils import iso
|
||||
from app.youtube.client import YouTubeClient, YouTubeError
|
||||
from app.refusal import refused
|
||||
@@ -293,3 +305,214 @@ def get_video_detail(
|
||||
items[0].get("contentDetails", {}).get("duration")
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# --- Google Takeout ------------------------------------------------------------------------------
|
||||
|
||||
|
||||
# One upload at a time per user, ten an hour, and two at once across every user: a file's bytes are
|
||||
# held whole while it is read (its text a window at a time), so these bound what the accounts can
|
||||
# make the process hold. The section uploads its files one after another.
|
||||
_takeout_limiter = RateLimiter(max_events=10, window_seconds=3600)
|
||||
_takeout_uploading: set[int] = set()
|
||||
_takeout_uploading_guard = threading.Lock()
|
||||
_TAKEOUT_AT_ONCE = 2
|
||||
|
||||
|
||||
async def _takeout_body(
|
||||
request: Request, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> AsyncIterator[bytes]:
|
||||
# The sign-in check has run, and it was a SELECT: hand the connection back before a file of
|
||||
# megabytes arrives, so a slow upload does not hold one `idle in transaction`. The user's id is
|
||||
# taken first: the release expires the user, and reading it after would reload it, taking a
|
||||
# connection straight back out.
|
||||
uid = user.id
|
||||
release_read_connection(db)
|
||||
with _takeout_uploading_guard:
|
||||
busy = uid in _takeout_uploading
|
||||
full = not busy and len(_takeout_uploading) >= _TAKEOUT_AT_ONCE
|
||||
taken = not busy and not full
|
||||
if taken:
|
||||
_takeout_uploading.add(uid)
|
||||
if not taken or not _takeout_limiter.allow(str(uid)):
|
||||
if taken:
|
||||
with _takeout_uploading_guard:
|
||||
_takeout_uploading.discard(uid)
|
||||
# Read to the end before answering: refused mid-upload, the browser reports a network
|
||||
# failure instead of the refusal (capped_stream_to says why).
|
||||
await drain_body(request)
|
||||
if busy:
|
||||
raise refused(429, "takeout_busy")
|
||||
if full:
|
||||
raise refused(429, "takeout_server_busy")
|
||||
raise refused(429, "takeout_cooldown")
|
||||
try:
|
||||
yield await capped_body(request, takeout.MAX_BYTES)
|
||||
finally:
|
||||
with _takeout_uploading_guard:
|
||||
_takeout_uploading.discard(uid)
|
||||
|
||||
|
||||
@router.post("/takeout")
|
||||
def upload_takeout(
|
||||
# The sign-in check FIRST: the body's own dependency needs the signed-in user, so a stranger's
|
||||
# file is refused before a byte of it is read.
|
||||
user: User = Depends(current_user),
|
||||
body: bytes = Depends(_takeout_body),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
"""One file of the user's Google Takeout export — the watch history or the Watch later list.
|
||||
Its videos are kept and the import runs in the background; its outcome reaches the user's inbox."""
|
||||
try:
|
||||
parsed = takeout.parse(body)
|
||||
except takeout.TakeoutError as exc:
|
||||
if exc.reason == "no_videos":
|
||||
raise refused(422, "takeout_no_videos") from None
|
||||
if exc.reason == "too_many":
|
||||
raise refused(422, "takeout_list_too_long", limit=takeout.WATCH_LATER_MAX) from None
|
||||
raise refused(422, "takeout_unreadable") from None
|
||||
try:
|
||||
takeout.stage(db, user, parsed)
|
||||
except takeout.TooMany:
|
||||
db.rollback()
|
||||
raise refused(422, "takeout_too_many", limit=takeout.ROWS_MAX) from None
|
||||
db.commit()
|
||||
scheduler.run_ad_hoc(
|
||||
scheduler.TAKEOUT_IMPORT,
|
||||
partial(takeout.run_import, user.id),
|
||||
actor_id=user.id,
|
||||
target=takeout.run_target(user.id),
|
||||
notify=True,
|
||||
)
|
||||
return {
|
||||
"ok": True,
|
||||
"kind": parsed.kind,
|
||||
"videos": len(parsed.videos),
|
||||
"entries": parsed.entries,
|
||||
"ads": parsed.ads,
|
||||
"other": parsed.other,
|
||||
"malformed": parsed.malformed,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/takeout")
|
||||
def takeout_status(user: User = Depends(current_user), db: Session = Depends(get_db)):
|
||||
"""What the user's Takeout import has done: its last run (none before the first upload), and
|
||||
every video kept from the export, by kind and outcome — a video the catalog did not know yet
|
||||
counts as "waiting" until it does, and is marked then."""
|
||||
target = takeout.run_target(user.id)
|
||||
run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, target)
|
||||
return {
|
||||
# Queued and not started: its run row is not written yet, and the last run is still the
|
||||
# previous one, so the page waits for it rather than taking that one for this upload's.
|
||||
"queued": scheduler.ad_hoc_queued(scheduler.TAKEOUT_IMPORT, target),
|
||||
"max_bytes": takeout.MAX_BYTES,
|
||||
"run": None
|
||||
if run is None
|
||||
else {
|
||||
"status": run.status,
|
||||
"started_at": iso(run.started_at),
|
||||
"finished_at": iso(run.finished_at),
|
||||
"result": run.result,
|
||||
"failure": run.failure,
|
||||
},
|
||||
**queries.takeout_summary(db, user),
|
||||
}
|
||||
|
||||
|
||||
# --- channels watched but not followed -----------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/takeout/channels")
|
||||
def watched_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict:
|
||||
"""The channels the user's Takeout history shows them watching often but not following: each
|
||||
with how many of its videos over five minutes they watched, and when last. `pending` is how many
|
||||
more channels may join once the import has asked YouTube for their videos' lengths or their
|
||||
names, so a list still being made does not read as the whole of it; `imported` tells "no history
|
||||
uploaded" from "nothing to suggest"."""
|
||||
return {
|
||||
"imported": queries.has_watch_history(db, user),
|
||||
"pending": queries.watched_channels_pending(db, user),
|
||||
"channels": [
|
||||
{
|
||||
"id": ch.id,
|
||||
"title": ch.title,
|
||||
"handle": ch.handle,
|
||||
"thumbnail_url": ch.thumbnail_url,
|
||||
"subscriber_count": ch.subscriber_count,
|
||||
"video_count": ch.video_count,
|
||||
"description": ch.description,
|
||||
"watched": int(n),
|
||||
"last_watched_at": iso(last),
|
||||
}
|
||||
for ch, n, last in queries.watched_channels(db, user)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@router.post("/takeout/channels/{channel_id}/videos")
|
||||
def watched_channel_videos(
|
||||
channel_id: str, user: User = Depends(current_user), db: Session = Depends(get_db)
|
||||
) -> dict:
|
||||
"""The videos of a channel that the user's list counts, ready to play, last watched first. The
|
||||
channel is explored for the user (as its page does), and the videos the catalog lacks come in as
|
||||
the exploration's: the player needs them in the catalog, the shared Library does not get them,
|
||||
and the cleanup reclaims them once nobody keeps them. Of a channel someone follows they come in
|
||||
as the catalog's own, as its uploads do. `gone`: how many YouTube no longer has."""
|
||||
channel = catalog_queries.channel(db, channel_id)
|
||||
if channel is None:
|
||||
raise not_found("channel")
|
||||
if catalog_queries.is_blocked(db, user, channel_id):
|
||||
raise refused(403, "channel_blocked")
|
||||
counted, gone = queries.watched_channel_videos(db, user, channel_id)
|
||||
ids = [vid for vid, _ in counted]
|
||||
known = queries.known_video_ids(db, ids)
|
||||
missing = [vid for vid in ids if vid not in known]
|
||||
if missing and quota.backfill_reserve_reached():
|
||||
raise refused(429, "quota_reserve")
|
||||
catalog_service.explore(db, user, channel)
|
||||
db.commit()
|
||||
if missing:
|
||||
try:
|
||||
with (
|
||||
quota.attribute(user.id, quota.QuotaAction.CHANNELS_EXPLORE),
|
||||
YouTubeClient(db, user) as yt,
|
||||
):
|
||||
ensure_videos(db, yt, missing, explore=channel.from_explore)
|
||||
except YouTubeError as exc:
|
||||
raise youtube_refusal(exc) from exc
|
||||
catalog_service.settle_explored(db, channel)
|
||||
db.commit()
|
||||
items = queries.feed_rows_by_ids(db, user, ids)
|
||||
watched_at = dict(counted)
|
||||
for item in items:
|
||||
item["takeout_watched_at"] = iso(watched_at[item["id"]])
|
||||
# A video YouTube still had when its length was asked may have gone since.
|
||||
return {"ok": True, "items": items, "gone": gone + len(ids) - len(items)}
|
||||
|
||||
|
||||
@router.put("/takeout/channels/{channel_id}/dismissal")
|
||||
def dismiss_watched_channel(
|
||||
channel_id: str = Path(max_length=32),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Put a channel out of the user's list (it stays out until restored). Only one their own history
|
||||
shows them watching: a made-up id would only pile up rows every list read then has to skip."""
|
||||
if not queries.watched_in_history(db, user, channel_id):
|
||||
raise not_found("channel")
|
||||
service.dismiss_watched_channel(db, user, channel_id)
|
||||
db.commit()
|
||||
return {"ok": True, "channel_id": channel_id}
|
||||
|
||||
|
||||
@router.delete("/takeout/channels/{channel_id}/dismissal")
|
||||
def restore_watched_channel(
|
||||
channel_id: str = Path(max_length=32),
|
||||
user: User = Depends(current_user),
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
"""Let a channel the user put out of the list back on it."""
|
||||
service.restore_watched_channel(db, user, channel_id)
|
||||
db.commit()
|
||||
return {"ok": True, "channel_id": channel_id}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
Admin-only and read-only — rows leave the table through the retention job
|
||||
(`job_run_retention_days`). Filtered on the server and paged by `_keyset`, like the server log and
|
||||
the audit log beside it on the activity page. A row names what a reader needs without a second
|
||||
request: the channel a user's ad-hoc run worked on, and how many log lines the run left (only
|
||||
request: what a user's ad-hoc run worked on (the channel, the importing user), and how many log lines the run left (only
|
||||
WARNING and up are stored, so most runs have none — the count says so before the click).
|
||||
"""
|
||||
|
||||
@@ -116,8 +116,12 @@ def list_runs(
|
||||
limit,
|
||||
lambda r: r.started_at,
|
||||
)
|
||||
channels = {r.target for r in runs if r.job in scheduler.AD_HOC_JOBS and r.target}
|
||||
channels = {r.target for r in runs if r.job == scheduler.CHANNEL_BACKFILL and r.target}
|
||||
titles = catalog_queries.channel_titles(db, channels)
|
||||
for target in {r.target for r in runs if r.job == scheduler.TAKEOUT_IMPORT and r.target}:
|
||||
title = scheduler.ad_hoc_target_title(db, scheduler.TAKEOUT_IMPORT, target)
|
||||
if title is not None:
|
||||
titles[target] = title
|
||||
lines: dict[int, int] = {}
|
||||
if runs:
|
||||
counted = db.execute(
|
||||
|
||||
+160
-41
@@ -22,6 +22,7 @@ from app.plex.tmdb import run_people_enrich
|
||||
from app.plex.watch_sync import run_plex_watch_reconcile, run_plex_watch_sync
|
||||
from app.notifications import create_notification
|
||||
from app.runstatus import RunStatus, RunTrigger
|
||||
from app.reader import takeout
|
||||
from app.state import is_sync_paused
|
||||
from app.sync.autotag import run_autotag_all
|
||||
from app.sync.explore import purge_ephemeral
|
||||
@@ -118,8 +119,9 @@ def _notify_done(
|
||||
summary: str | None,
|
||||
result: dict | None = None,
|
||||
) -> None:
|
||||
"""Post a completion notice to the triggering admin's inbox (manual runs only). Best
|
||||
effort — a notification failure must never mask the job's own outcome."""
|
||||
"""Post a completion notice to the inbox of who started the run: an admin's manual run, or a
|
||||
user's ad-hoc run that asks for one. Best effort — a notification failure must never mask the
|
||||
job's own outcome."""
|
||||
try:
|
||||
create_notification(
|
||||
db,
|
||||
@@ -272,7 +274,6 @@ def scheduler_snapshot(db) -> dict:
|
||||
open_runs = _open_run_ids()
|
||||
recent = jobruns.recent_runs(db, [*JOB_INTERVALS, *AD_HOC_JOBS], RECENT_RUNS)
|
||||
open_runs |= _open_run_ids()
|
||||
going_id = max((r for r, j in open_runs.items() if j == CHANNEL_BACKFILL), default=None)
|
||||
jobs = []
|
||||
for job_id, default_interval in JOB_INTERVALS.items():
|
||||
a = acts.get(job_id, {})
|
||||
@@ -288,30 +289,11 @@ def scheduler_snapshot(db) -> dict:
|
||||
)
|
||||
# Work a user starts on one target: no interval, no "run now" (it needs its target), but its
|
||||
# runs are shown beside the jobs' — it spends the same quota. Its one in-memory guard is the
|
||||
# target's own (sync/runner), so "running" is the runs this process has open: two channels can
|
||||
# backfill at once, so a run still going counts even when a newer one already ended. That run then
|
||||
# leads the row, so its channel, its "no outcome yet" and its dot are one run's, never the
|
||||
# newer run's outcome under the older one's channel.
|
||||
backfills = recent.get(CHANNEL_BACKFILL, [])
|
||||
going = db.get(JobRun, going_id) if going_id is not None else None
|
||||
if going is not None and going.status != RunStatus.RUNNING:
|
||||
going = None # its outcome is written; only the bookkeeping has not caught up
|
||||
if going is not None:
|
||||
backfills = [going, *(r for r in backfills if r.id != going.id)][:RECENT_RUNS]
|
||||
shown = backfills[0] if backfills else None
|
||||
ad_hoc = [
|
||||
{
|
||||
"id": CHANNEL_BACKFILL,
|
||||
"running": going is not None,
|
||||
"target": shown.target if shown else None,
|
||||
"target_title": (
|
||||
catalog_queries.channel_title(db, shown.target)
|
||||
if shown is not None and shown.target
|
||||
else None
|
||||
),
|
||||
**_last_runs(backfills, open_runs),
|
||||
}
|
||||
]
|
||||
# target's own (sync/runner, the import's per-user lock), so "running" is the runs this process
|
||||
# has open: two targets can run at once, so a run still going counts even when a newer one already
|
||||
# ended. That run then leads the row, so its target, its "no outcome yet" and its dot are one
|
||||
# run's, never the newer run's outcome under the older one's target.
|
||||
ad_hoc = [_ad_hoc_row(db, job, recent.get(job, []), open_runs) for job in AD_HOC_JOBS]
|
||||
return {
|
||||
"running_here": running_here,
|
||||
"enabled": settings.scheduler_enabled,
|
||||
@@ -333,6 +315,27 @@ def _seen(run, open_runs: dict[int, str]) -> RunStatus:
|
||||
return RunStatus(run.status)
|
||||
|
||||
|
||||
def _ad_hoc_row(db, job: str, runs: list[JobRun], open_runs: dict[int, str]) -> dict:
|
||||
going_id = max((r for r, j in open_runs.items() if j == job), default=None)
|
||||
going = db.get(JobRun, going_id) if going_id is not None else None
|
||||
if going is not None and going.status != RunStatus.RUNNING:
|
||||
going = None # its outcome is written; only the bookkeeping has not caught up
|
||||
if going is not None:
|
||||
runs = [going, *(r for r in runs if r.id != going.id)][:RECENT_RUNS]
|
||||
shown = runs[0] if runs else None
|
||||
return {
|
||||
"id": job,
|
||||
"running": going is not None,
|
||||
"target": shown.target if shown else None,
|
||||
"target_title": (
|
||||
ad_hoc_target_title(db, job, shown.target)
|
||||
if shown is not None and shown.target
|
||||
else None
|
||||
),
|
||||
**_last_runs(runs, open_runs),
|
||||
}
|
||||
|
||||
|
||||
def _last_runs(runs: list, open_runs: dict[int, str]) -> dict:
|
||||
"""A row's run fields from its newest runs (newest first): how the last one went, and the few
|
||||
before it for the history dots."""
|
||||
@@ -402,7 +405,12 @@ def _autotag_job() -> None:
|
||||
|
||||
|
||||
def _shorts_job() -> None:
|
||||
_job("shorts", run_shorts)
|
||||
def work(db):
|
||||
# A video waiting in someone's Takeout import becomes markable the moment the probe has
|
||||
# said it is not a Short, so the import's waiting rows are settled right after it.
|
||||
return run_shorts(db) | takeout.apply_waiting(db)
|
||||
|
||||
_job("shorts", work)
|
||||
|
||||
|
||||
def _subscriptions_job() -> None:
|
||||
@@ -608,48 +616,140 @@ def trigger_all(actor_id: int | None = None) -> list[str]:
|
||||
# one channel. It runs on the same scheduler as a recorded run (trigger `ad_hoc`), so what it did and
|
||||
# spent is in `job_runs` like any job's, not a background task only the log ever heard of.
|
||||
CHANNEL_BACKFILL = "channel_backfill"
|
||||
# A user's Google Takeout import (`reader.takeout`): its target is the user.
|
||||
TAKEOUT_IMPORT = "takeout_import"
|
||||
# Every ad-hoc job id: the page names each one (a frontend contract test reads this tuple).
|
||||
AD_HOC_JOBS = (CHANNEL_BACKFILL,)
|
||||
AD_HOC_JOBS = (CHANNEL_BACKFILL, TAKEOUT_IMPORT)
|
||||
# What each one's target is called on the admin's pages: a channel's title, an importing user's name.
|
||||
_TARGET_TITLES: dict[str, Callable[..., str | None]] = {
|
||||
CHANNEL_BACKFILL: catalog_queries.channel_title,
|
||||
TAKEOUT_IMPORT: takeout.target_title,
|
||||
}
|
||||
|
||||
|
||||
def run_ad_hoc(job: str, fn: Callable[[], dict], *, actor_id: int, target: str) -> None:
|
||||
def ad_hoc_target_title(db, job: str, target: str) -> str | None:
|
||||
"""The name of an ad-hoc run's target (None for a job with none, or a target since gone)."""
|
||||
title = _TARGET_TITLES.get(job)
|
||||
return title(db, target) if title is not None else None
|
||||
|
||||
|
||||
# The ad-hoc runs added and not yet recorded, `job@target`: between the two the target's newest
|
||||
# run row is still the previous run's, so a page watching it asks `ad_hoc_queued` too.
|
||||
_queued: set[str] = set()
|
||||
_queued_guard = threading.Lock()
|
||||
# What the scheduler is told to run is only the key: the request itself waits here, the newest one
|
||||
# per key, until a fire of that key takes it. A request made while its key's run is under way is so
|
||||
# kept for the run after it, which that run asks for as it ends (`_running`): the scheduler would
|
||||
# drop a second fire of an id whose run has not returned (one instance per job id).
|
||||
_pending: dict[str, tuple[Callable[[], dict], int, bool]] = {}
|
||||
_running: set[str] = set()
|
||||
# Two at once, for the moment between a run letting go of its key and its thread returning: a fire
|
||||
# then is what takes the next request, and is not dropped.
|
||||
_AD_HOC_INSTANCES = 2
|
||||
|
||||
|
||||
def ad_hoc_queued(job: str, target: str) -> bool:
|
||||
"""Whether a run of `job` on `target` is queued and its run row not written yet."""
|
||||
with _queued_guard:
|
||||
return f"{job}@{target}" in _queued
|
||||
|
||||
|
||||
def run_ad_hoc(
|
||||
job: str, fn: Callable[[], dict], *, actor_id: int, target: str, notify: bool = False
|
||||
) -> None:
|
||||
"""Run `fn` now as `actor_id`'s run of `job` on `target`, a one-shot job on the scheduler. Its
|
||||
own exclusion (if any) lives in `fn`, which says so in its result; one id per job and target, so
|
||||
a second request before the first has even started is that same run."""
|
||||
a second request before the first has even started is that same run, and one made while it runs
|
||||
is the run after it. `notify` posts its outcome to the user's inbox, for work whose page does not
|
||||
watch it run."""
|
||||
if job not in AD_HOC_JOBS:
|
||||
raise KeyError(job) # an unnamed job would show on the page as its bare id
|
||||
sched = _scheduler
|
||||
if sched is None:
|
||||
if _scheduler is None:
|
||||
# Only between shutdown and exit: start_scheduler runs at every API startup.
|
||||
raise RuntimeError("the scheduler is not running")
|
||||
key = f"{job}@{target}"
|
||||
with _queued_guard:
|
||||
_pending[key] = (fn, actor_id, notify)
|
||||
_queued.add(key)
|
||||
running = key in _running
|
||||
if running:
|
||||
log.info("ad-hoc %s for %s runs again once the run under way ends", job, target)
|
||||
return
|
||||
_fire(job, target)
|
||||
|
||||
|
||||
def _fire(job: str, target: str) -> None:
|
||||
"""Add the one-shot that takes `job@target`'s pending request. Raises (the request dropped) only
|
||||
when the scheduler cannot take it."""
|
||||
key = f"{job}@{target}"
|
||||
sched = _scheduler
|
||||
try:
|
||||
if sched is None:
|
||||
raise RuntimeError("the scheduler is not running")
|
||||
sched.add_job(
|
||||
_ad_hoc,
|
||||
_ad_hoc_fired,
|
||||
"date",
|
||||
args=(job, fn, actor_id, target),
|
||||
id=f"{job}@{target}",
|
||||
args=(job, target),
|
||||
id=key,
|
||||
misfire_grace_time=None,
|
||||
max_instances=_AD_HOC_INSTANCES,
|
||||
)
|
||||
# silent-ok: not a failure — the run for this target is already queued, and it does this work.
|
||||
# silent-ok: not a failure — a fire of this key is already queued, and it takes this request.
|
||||
except ConflictingIdError:
|
||||
log.info("ad-hoc %s for %s already queued", job, target)
|
||||
except Exception:
|
||||
with _queued_guard:
|
||||
_pending.pop(key, None)
|
||||
_queued.discard(key)
|
||||
raise
|
||||
|
||||
|
||||
def _ad_hoc(job: str, fn: Callable[[], dict], actor_id: int, target: str) -> None:
|
||||
def _ad_hoc_fired(job: str, target: str) -> None:
|
||||
key = f"{job}@{target}"
|
||||
with _queued_guard:
|
||||
request = _pending.pop(key, None)
|
||||
if request is None:
|
||||
return # an earlier fire took it
|
||||
_running.add(key)
|
||||
fn, actor_id, notify = request
|
||||
try:
|
||||
_ad_hoc(job, fn, actor_id, target, notify)
|
||||
finally:
|
||||
with _queued_guard:
|
||||
_running.discard(key)
|
||||
again = key in _pending
|
||||
if again:
|
||||
try:
|
||||
_fire(job, target)
|
||||
except Exception:
|
||||
# The request made meanwhile is lost with the scheduler: shut down, nothing runs.
|
||||
log.exception("ad-hoc %s for %s not run again", job, target)
|
||||
|
||||
|
||||
def _ad_hoc(
|
||||
job: str, fn: Callable[[], dict], actor_id: int, target: str, notify: bool = False
|
||||
) -> None:
|
||||
key = f"{job}@{target}"
|
||||
try:
|
||||
run_id = jobruns.start(job, RunTrigger.AD_HOC, actor_id, target)
|
||||
except Exception:
|
||||
# No row to tell it in: the database is what failed. The log is the record.
|
||||
log.exception("ad-hoc %s for %s not started: its run could not be recorded", job, target)
|
||||
return
|
||||
finally:
|
||||
# Recorded (or never will be): the run row now speaks for it — unless another request came
|
||||
# meanwhile, which is still queued.
|
||||
with _queued_guard:
|
||||
if key not in _pending:
|
||||
_queued.discard(key)
|
||||
_opened(run_id, job)
|
||||
try:
|
||||
_ad_hoc_work(job, fn, run_id)
|
||||
_ad_hoc_work(job, fn, run_id, actor_id if notify else None)
|
||||
finally:
|
||||
_closed(run_id)
|
||||
|
||||
|
||||
def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None:
|
||||
def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int, notify_id: int | None) -> None:
|
||||
with logs.bind(run_id=run_id):
|
||||
# As in _run: only the work is inside the try.
|
||||
try:
|
||||
@@ -658,8 +758,23 @@ def _ad_hoc_work(job: str, fn: Callable[[], dict], run_id: int) -> None:
|
||||
log.exception("job %s failed", job, extra=logs.event("scheduler.job.failed", job=job))
|
||||
err, failure = _failure(exc)
|
||||
_close(run_id, RunStatus.ERROR, error=err, failure=failure)
|
||||
# The user's notice names the failure, never the upstream's own text: that is an English
|
||||
# diagnostic, for the admin's run row and the log, not a user's inbox.
|
||||
_notify_ad_hoc(
|
||||
notify_id, job, RunStatus.ERROR, None, failure or {"failure": "unexpected"}
|
||||
)
|
||||
return
|
||||
_finish_ok(job, run_id, result)
|
||||
summary, data = _finish_ok(job, run_id, result)
|
||||
_notify_ad_hoc(notify_id, job, RunStatus.OK, summary, data)
|
||||
|
||||
|
||||
def _notify_ad_hoc(
|
||||
user_id: int | None, job: str, status: RunStatus, summary: str | None, result: dict | None
|
||||
) -> None:
|
||||
if user_id is None:
|
||||
return
|
||||
with SessionLocal() as db:
|
||||
_notify_done(db, user_id, job, status, summary, result)
|
||||
|
||||
|
||||
# Room for every job at once (a "run all") beside a few ad-hoc runs: APScheduler's default pool of
|
||||
@@ -700,3 +815,7 @@ def shutdown_scheduler() -> None:
|
||||
if _scheduler is not None:
|
||||
_scheduler.shutdown(wait=False)
|
||||
_scheduler = None
|
||||
with _queued_guard:
|
||||
# What it held will not run.
|
||||
_queued.clear()
|
||||
_pending.clear()
|
||||
|
||||
@@ -14,6 +14,7 @@ from sqlalchemy import delete, or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs, sysconfig
|
||||
from app.catalog import service as catalog_service
|
||||
from app.catalog.models import Channel, ExploredChannel, SearchFind, Subscription, Video
|
||||
from app.reader import queries as reader_queries
|
||||
from app.sync.videos import insert_stubs, stub_from_playlist_item, apply_video_details
|
||||
@@ -66,11 +67,13 @@ def _enrich_new(db: Session, yt: YouTubeClient, video_ids: list[str]) -> int:
|
||||
def explore_ingest_page(
|
||||
db: Session, yt: YouTubeClient, channel: Channel, page_token: str | None = None
|
||||
) -> dict:
|
||||
"""Ingest one page (up to 50, newest-first) of the channel's uploads, marking the new
|
||||
videos via_explore, then enrich them. Returns the count ingested and the next page token
|
||||
(None when the back-catalogue is exhausted) so the caller can offer "load more". Stateless
|
||||
pagination: the caller round-trips the token, so the channel's own backfill cursor (used by
|
||||
the organic deep backfill) is never touched."""
|
||||
"""Ingest one page (up to 50, newest-first) of the channel's uploads, marking the new videos
|
||||
via_explore while the channel is an exploration's (`from_explore`): of a channel someone follows
|
||||
they are its uploads like any other, and a flag there would never be cleared, since subscribing
|
||||
clears only an explored channel's. Then enrich them. Returns the count ingested and the next
|
||||
page token (None when the back-catalogue is exhausted) so the caller can offer "load more".
|
||||
Stateless pagination: the caller round-trips the token, so the channel's own backfill cursor
|
||||
(used by the organic deep backfill) is never touched."""
|
||||
if not channel.uploads_playlist_id:
|
||||
return {"ingested": 0, "next_page_token": None}
|
||||
try:
|
||||
@@ -86,9 +89,10 @@ def explore_ingest_page(
|
||||
for item in data.get("items", []):
|
||||
stub = stub_from_playlist_item(item, channel.id)
|
||||
if stub is not None:
|
||||
stub["via_explore"] = True
|
||||
stub["via_explore"] = channel.from_explore
|
||||
stubs.append(stub)
|
||||
inserted = insert_stubs(db, stubs)
|
||||
catalog_service.settle_explored(db, channel)
|
||||
self_enriched = _enrich_new(db, yt, [s["id"] for s in stubs])
|
||||
log.info(
|
||||
"explore ingest channel=%s ingested=%s enriched=%s", channel.id, inserted, self_enriched
|
||||
@@ -105,7 +109,9 @@ def purge_unkept_explored(db: Session) -> dict:
|
||||
- a live ExploredChannel row (someone explored it within the grace window — channel-level), or
|
||||
- a video has per-user state: watched/in-progress (VideoState), in a playlist (PlaylistItem),
|
||||
or surfaced by the user's own search (SearchFind) — these survive even if the channel goes.
|
||||
A channel row is dropped only once it holds no videos at all. 0 quota."""
|
||||
A channel row is dropped only once it holds no videos at all, and not while someone's Takeout
|
||||
history shows it watched (their "watched but not followed" list names it from that row).
|
||||
0 quota."""
|
||||
grace_days = sysconfig.get_int(db, "explore_grace_days")
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=grace_days)
|
||||
|
||||
@@ -135,7 +141,9 @@ def purge_unkept_explored(db: Session) -> dict:
|
||||
|
||||
has_videos = select(Video.id).where(Video.channel_id == Channel.id).exists()
|
||||
channels_deleted = db.execute(
|
||||
delete(Channel).where(Channel.id.in_(orphan_ids), ~has_videos)
|
||||
delete(Channel).where(
|
||||
Channel.id.in_(orphan_ids), ~has_videos, ~reader_queries.in_a_watch_history(Channel.id)
|
||||
)
|
||||
).rowcount
|
||||
db.commit()
|
||||
return {
|
||||
|
||||
@@ -8,6 +8,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app import logs
|
||||
from app.models import User
|
||||
from app.catalog import service as catalog_service
|
||||
from app.catalog.models import Channel, Subscription
|
||||
from app.utils import normalize_description
|
||||
from app.youtube.client import YouTubeClient, YouTubeError, best_thumbnail
|
||||
@@ -109,6 +110,21 @@ def apply_channel_details(db: Session, items: list[dict], asked: Iterable[str])
|
||||
channel.details_synced_at = now
|
||||
|
||||
|
||||
def describe_channels(db: Session, yt: YouTubeClient, channel_ids: list[str]) -> None:
|
||||
"""Make sure each given channel is in the catalog with its details: a channel it lacks is added
|
||||
as a stub, then all of them are asked for (`channels.list`, 1 unit a 50). Flushes; the caller
|
||||
commits."""
|
||||
if not channel_ids:
|
||||
return
|
||||
have = set(db.execute(select(Channel.id).where(Channel.id.in_(channel_ids))).scalars())
|
||||
for cid in dict.fromkeys(channel_ids):
|
||||
if cid not in have:
|
||||
db.add(Channel(id=cid))
|
||||
db.flush()
|
||||
apply_channel_details(db, yt.get_channels(channel_ids), channel_ids)
|
||||
db.flush()
|
||||
|
||||
|
||||
def should_prune(fetched_count: int, existing_count: int) -> bool:
|
||||
"""Whether an import may delete the local subscriptions missing from this fetch.
|
||||
|
||||
@@ -182,6 +198,10 @@ def import_subscriptions(db: Session, user: User) -> dict:
|
||||
db.add(sub_row)
|
||||
sub_row.yt_subscription_id = sub.get("yt_subscription_id")
|
||||
db.flush()
|
||||
# A follow made on youtube.com keeps an explored channel as a follow made here does.
|
||||
explored = select(Channel).where(Channel.id.in_(fetched), Channel.from_explore.is_(True))
|
||||
for channel in db.execute(explored).scalars():
|
||||
catalog_service.settle_explored(db, channel)
|
||||
|
||||
# Prune subscriptions removed on YouTube — but only when this fetch is trustworthy
|
||||
# (see `should_prune`).
|
||||
|
||||
@@ -420,11 +420,15 @@ def run_shorts_classification(db: Session, limit: int | None = None) -> dict:
|
||||
return {"probed": probed, "shorts": shorts}
|
||||
|
||||
|
||||
def ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
|
||||
def ensure_videos(
|
||||
db: Session, yt: YouTubeClient, video_ids: list[str], *, explore: bool = False
|
||||
) -> None:
|
||||
"""Make sure every given video id exists in the catalog, fetching + ingesting the
|
||||
missing ones (and a stub channel for any unknown channel). Deleted/private videos that
|
||||
the API won't return are simply left out. The catalog's write for a playlist sync that names
|
||||
videos it has never seen; commits."""
|
||||
videos it has never seen; commits. `explore` adds the new ones as an exploration's
|
||||
(`via_explore`): visible to who explores or follows their channel, and reclaimed by the cleanup
|
||||
once nobody keeps them."""
|
||||
if not video_ids:
|
||||
return
|
||||
have = set(db.execute(select(Video.id).where(Video.id.in_(video_ids))).scalars().all())
|
||||
@@ -458,7 +462,12 @@ def ensure_videos(db: Session, yt: YouTubeClient, video_ids: list[str]) -> None:
|
||||
seen.add(vid)
|
||||
# published_at is NOT NULL (0068); apply_video_details re-fills a missing date, but seed a now()
|
||||
# fallback here too so the constructor never carries an explicit NULL.
|
||||
v = Video(id=vid, channel_id=cid, published_at=parse_dt(sn.get("publishedAt")) or now)
|
||||
v = Video(
|
||||
id=vid,
|
||||
channel_id=cid,
|
||||
published_at=parse_dt(sn.get("publishedAt")) or now,
|
||||
via_explore=explore,
|
||||
)
|
||||
apply_video_details(v, it)
|
||||
v.enriched_at = now
|
||||
db.add(v)
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
# bumped in place instead of a full regeneration, which would move every other package too — then
|
||||
# `pip install -r requirements.lock && pip check` must pass. In place so far: urllib3 2.7.0 -> 2.8.0 (PYSEC-2026-4175/4176/4177); yt-dlp 2026.7.4 -> 2026.8.19
|
||||
# (it marks the clients that need a stream PO token, so the downloader stops fetching into a 403);
|
||||
# certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2).
|
||||
# certifi 2026.7.22 -> 2026.10.19 (GHSA-6q48-6c7w-9gw2); brotli 1.2.0 -> 1.2.1 (PYSEC-2026-4322).
|
||||
APScheduler==3.11.3
|
||||
Authlib==1.7.2
|
||||
Mako==1.4.1
|
||||
@@ -21,7 +21,7 @@ anyio==4.14.2
|
||||
argon2-cffi-bindings==25.1.0
|
||||
argon2-cffi==25.1.0
|
||||
bgutil-ytdlp-pot-provider==1.3.1
|
||||
brotli==1.2.0
|
||||
brotli==1.2.1
|
||||
certifi==2026.10.19
|
||||
cffi==2.1.1
|
||||
charset-normalizer==3.5.1
|
||||
|
||||
@@ -327,3 +327,15 @@ def _reset_log_erasures():
|
||||
from app import logs
|
||||
|
||||
logs._erased.clear()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_ad_hoc_queue():
|
||||
"""The scheduler's ad-hoc queue is module-level too: a request a test left pending would read as
|
||||
a later test's user's import still queued (ids restart from 1)."""
|
||||
from app import scheduler
|
||||
|
||||
with scheduler._queued_guard:
|
||||
scheduler._queued.clear()
|
||||
scheduler._pending.clear()
|
||||
scheduler._running.clear()
|
||||
|
||||
@@ -159,6 +159,15 @@ def test_the_run_list_filters_pages_and_names_the_channel_and_the_log_lines(db,
|
||||
assert facets["total"] == 4
|
||||
|
||||
|
||||
def test_the_run_list_names_the_user_a_takeout_import_ran_for(db, make_user, as_user):
|
||||
admin = make_user(role="admin")
|
||||
importer = make_user("importer@example.com")
|
||||
imported = _run(db, "takeout_import", "ok", importer.id, T0, f"user:{importer.id}", "ad_hoc")
|
||||
db.commit()
|
||||
(row,) = as_user(admin).get("/api/admin/runs").json()["items"]
|
||||
assert (row["id"], row["target_title"]) == (imported, "importer@example.com")
|
||||
|
||||
|
||||
def test_the_quota_breakdown_covers_its_range_and_one_account(db, make_user, as_user):
|
||||
admin = make_user(role="admin")
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
@@ -23,6 +23,7 @@ from app import bodylimit, debugsession, sysconfig
|
||||
from app.config import settings
|
||||
from app.models import SystemConfig
|
||||
from app.bodylimit import DEFAULT_MAX_BODY_BYTES, BodyLimitMiddleware
|
||||
from app.reader import takeout
|
||||
from app.routes import diagnostics as diagnostics_routes
|
||||
|
||||
SMALL = b"x" * 16
|
||||
@@ -221,6 +222,7 @@ def test_the_default_ceiling_clears_the_largest_body_a_default_route_takes():
|
||||
CAPPED_BODY_SITES = {
|
||||
("diagnostics.py", "_parsed_events", "capped_body", "BODY_MAX_BYTES"),
|
||||
("debug_sessions.py", "_image_body", "capped_body", "debugsession.SCREENSHOT_MAX_BYTES"),
|
||||
("feed.py", "_takeout_body", "capped_body", "takeout.MAX_BYTES"),
|
||||
("files.py", "spool_upload", "capped_stream_to", "cap"),
|
||||
}
|
||||
|
||||
@@ -368,6 +370,9 @@ def test_every_body_reading_site_is_known_and_fits_under_its_ceiling():
|
||||
# The upload site's limit is the sysconfig knob, which is bounded BY the edge ceiling rather
|
||||
# than being a constant to compare against — that pairing is its own test below.
|
||||
assert ("POST", "/api/files/upload") in bodylimit.ROUTE_MAX_BODY_BYTES
|
||||
# A Takeout file is larger than the default: without its ceiling every real history is a 413.
|
||||
assert takeout.MAX_BYTES > DEFAULT_MAX_BODY_BYTES
|
||||
assert bodylimit.ROUTE_MAX_BODY_BYTES[("POST", "/api/takeout")] == takeout.MAX_BYTES
|
||||
|
||||
|
||||
def test_release_read_connection_ends_the_transaction_it_was_called_for(db):
|
||||
|
||||
@@ -12,6 +12,7 @@ from datetime import datetime, timedelta, timezone
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.catalog import queries, search, service
|
||||
from app.db import SessionLocal
|
||||
from app.catalog.models import (
|
||||
BlockedChannel,
|
||||
Channel,
|
||||
@@ -22,6 +23,8 @@ from app.catalog.models import (
|
||||
)
|
||||
from app.reader import queries as reader_queries
|
||||
from app.reader import service as reader_service
|
||||
from app.sync import explore as sync_explore
|
||||
from app.sync import subscriptions as sync_subscriptions
|
||||
|
||||
|
||||
def _channel(db, cid="UCc", **kw):
|
||||
@@ -141,6 +144,89 @@ def test_subscribing_keeps_an_explored_channel_and_its_videos(db, make_user):
|
||||
assert queries.subscription(db, user, "UCc") is None
|
||||
|
||||
|
||||
def test_exploring_a_channel_someone_follows_brings_its_uploads_in_as_the_catalogs_own(
|
||||
db, make_user, monkeypatch
|
||||
):
|
||||
follower, explorer = make_user(), make_user()
|
||||
ch = _channel(db, uploads_playlist_id="UUc")
|
||||
db.add(Subscription(user_id=follower.id, channel_id="UCc"))
|
||||
db.commit()
|
||||
service.explore(db, explorer, ch)
|
||||
db.commit()
|
||||
|
||||
class _Yt:
|
||||
def get_playlist_items(self, playlist_id, page_token):
|
||||
item = {"contentDetails": {"videoId": "v1", "videoPublishedAt": "2026-05-01T00:00:00Z"}}
|
||||
return {"items": [item]}
|
||||
|
||||
monkeypatch.setattr(sync_explore, "_enrich_new", lambda session, yt, ids: 0)
|
||||
sync_explore.explore_ingest_page(db, _Yt(), ch)
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
# The channel is followed, so not the exploration's: a video flagged as one would stay flagged
|
||||
# (subscribing clears only an explored channel's) and its watched marks would wait for good.
|
||||
assert (ch.from_explore, db.get(Video, "v1").via_explore) == (False, False)
|
||||
|
||||
|
||||
def test_a_follow_pulled_in_from_youtube_makes_an_explored_channel_the_catalogs_own(
|
||||
db, make_user, monkeypatch
|
||||
):
|
||||
explorer, follower = make_user(), make_user()
|
||||
ch = _channel(db, details_synced_at=datetime.now(timezone.utc))
|
||||
service.explore(db, explorer, ch)
|
||||
db.commit()
|
||||
_video(db, "v1", via_explore=True)
|
||||
|
||||
class _Yt:
|
||||
def __init__(self, session, user):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def iter_subscriptions(self):
|
||||
yield {"channel_id": "UCc", "title": "UCc", "yt_subscription_id": "sub-UCc"}
|
||||
|
||||
def get_channels(self, ids):
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(sync_subscriptions, "YouTubeClient", _Yt)
|
||||
sync_subscriptions.import_subscriptions(db, follower)
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
# Followed on youtube.com is followed all the same: left flagged, the channel's later videos
|
||||
# came in as an exploration's, hidden from the Library and their watched marks waiting for good.
|
||||
assert (db.get(Channel, "UCc").from_explore, db.get(Video, "v1").via_explore) == (False, False)
|
||||
|
||||
|
||||
def test_a_follow_landing_while_an_explored_page_comes_in_leaves_none_of_it_flagged(
|
||||
db, make_user, monkeypatch
|
||||
):
|
||||
explorer, follower = make_user(), make_user()
|
||||
ch = _channel(db, uploads_playlist_id="UUc")
|
||||
service.explore(db, explorer, ch)
|
||||
db.commit()
|
||||
|
||||
class _Yt:
|
||||
def get_playlist_items(self, playlist_id, page_token):
|
||||
# Someone subscribes, in a request of their own, while YouTube answers this one.
|
||||
with SessionLocal() as other:
|
||||
service.record_subscription(other, follower, other.get(Channel, "UCc"), "sub-UCc")
|
||||
other.commit()
|
||||
item = {"contentDetails": {"videoId": "v1", "videoPublishedAt": "2026-05-01T00:00:00Z"}}
|
||||
return {"items": [item]}
|
||||
|
||||
monkeypatch.setattr(sync_explore, "_enrich_new", lambda session, yt, ids: 0)
|
||||
sync_explore.explore_ingest_page(db, _Yt(), ch)
|
||||
db.commit()
|
||||
db.expire_all()
|
||||
# The subscribe cleared what was flagged before it; a page that lands after it must not be.
|
||||
assert (db.get(Channel, "UCc").from_explore, db.get(Video, "v1").via_explore) == (False, False)
|
||||
|
||||
|
||||
def test_a_block_is_once_and_ends_the_exploration(db, make_user):
|
||||
user = make_user()
|
||||
ch = _channel(db)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Migration 0116's data step, DB-lane: an explored channel someone already follows becomes the
|
||||
catalog's own, and no video stays flagged as an exploration's on a channel that is not one.
|
||||
|
||||
The schema half is checked by test_migrations.py; a flag left on is invisible there.
|
||||
"""
|
||||
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import engine
|
||||
|
||||
|
||||
def test_followed_explored_channels_and_their_videos_are_the_catalogs_own(db, make_user):
|
||||
follower = make_user().id
|
||||
cfg = Config("alembic.ini")
|
||||
command.downgrade(cfg, "0115_takeout_channels")
|
||||
try:
|
||||
with engine.begin() as conn:
|
||||
|
||||
def channel(cid, explored):
|
||||
conn.execute(
|
||||
text("INSERT INTO channels (id, from_explore) VALUES (:c, :e)"),
|
||||
{"c": cid, "e": explored},
|
||||
)
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO videos (id, channel_id, published_at, via_explore)"
|
||||
" VALUES (:v, :c, now(), true)"
|
||||
),
|
||||
{"v": f"v-{cid}", "c": cid},
|
||||
)
|
||||
|
||||
channel("UCfollowed", True) # explored, then followed on youtube.com
|
||||
channel("UCexplored", True) # explored, nobody follows it: stays the exploration's
|
||||
channel("UCorganic", False) # a video flagged after a subscribe had cleared the channel
|
||||
conn.execute(
|
||||
text("INSERT INTO subscriptions (user_id, channel_id) VALUES (:u, 'UCfollowed')"),
|
||||
{"u": follower},
|
||||
)
|
||||
finally:
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
with engine.connect() as conn:
|
||||
channels = dict(conn.execute(text("SELECT id, from_explore FROM channels")).tuples().all())
|
||||
videos = dict(
|
||||
conn.execute(text("SELECT channel_id, via_explore FROM videos")).tuples().all()
|
||||
)
|
||||
assert channels == {"UCfollowed": False, "UCexplored": True, "UCorganic": False}
|
||||
assert videos == {"UCfollowed": False, "UCexplored": True, "UCorganic": False}
|
||||
@@ -6,6 +6,7 @@ through (at the top of _job): holding _activity_lock across the check-and-set me
|
||||
caller can hold a job's "running" claim at a time, and it's claimable again once released."""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
@@ -342,6 +343,29 @@ def test_an_ad_hoc_run_is_the_users_run_of_its_target_with_its_spend_and_failure
|
||||
assert (failed.status, failed.error) == ("error", "quota exceeded")
|
||||
|
||||
|
||||
def _ad_hoc_row(db, job: str) -> dict:
|
||||
rows = scheduler.scheduler_snapshot(db)["ad_hoc"]
|
||||
assert [r["id"] for r in rows] == list(scheduler.AD_HOC_JOBS) # every one has its row
|
||||
return next(r for r in rows if r["id"] == job)
|
||||
|
||||
|
||||
def test_a_takeout_import_has_its_row_named_by_the_importing_user(db, make_user):
|
||||
user = make_user("importer@example.com")
|
||||
user.display_name = "Ida Importer"
|
||||
db.commit()
|
||||
assert _ad_hoc_row(db, "takeout_import")["recent"] == []
|
||||
scheduler._ad_hoc("takeout_import", lambda: {"watched": 2}, user.id, f"user:{user.id}")
|
||||
|
||||
row = _ad_hoc_row(db, "takeout_import")
|
||||
assert (row["running"], row["target"], row["target_title"], row["status"]) == (
|
||||
False,
|
||||
f"user:{user.id}",
|
||||
"Ida Importer",
|
||||
"ok",
|
||||
)
|
||||
assert _ad_hoc_row(db, "channel_backfill")["recent"] == []
|
||||
|
||||
|
||||
def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, make_user):
|
||||
user = make_user("shown@example.com")
|
||||
db.add(Channel(id="UC_shown", title="BBC Earth"))
|
||||
@@ -349,7 +373,7 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m
|
||||
scheduler._ad_hoc("channel_backfill", lambda: {"skipped": "at capacity"}, user.id, "UC_none")
|
||||
scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 69}, user.id, "UC_shown")
|
||||
|
||||
(row,) = scheduler.scheduler_snapshot(db)["ad_hoc"]
|
||||
row = _ad_hoc_row(db, "channel_backfill")
|
||||
assert (row["id"], row["running"], row["target"], row["target_title"]) == (
|
||||
"channel_backfill",
|
||||
False,
|
||||
@@ -367,12 +391,12 @@ def test_the_page_shows_the_ad_hoc_runs_beside_the_jobs_with_their_channel(db, m
|
||||
seen = []
|
||||
scheduler._ad_hoc(
|
||||
"channel_backfill",
|
||||
lambda: seen.append(scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"]) or {},
|
||||
lambda: seen.append(_ad_hoc_row(db, "channel_backfill")["running"]) or {},
|
||||
user.id,
|
||||
"UC_shown",
|
||||
)
|
||||
assert seen == [True]
|
||||
assert scheduler.scheduler_snapshot(db)["ad_hoc"][0]["running"] is False
|
||||
assert _ad_hoc_row(db, "channel_backfill")["running"] is False
|
||||
|
||||
|
||||
def _finished(db, job) -> list[JobRun]:
|
||||
@@ -397,9 +421,46 @@ def test_a_second_ad_hoc_request_before_the_first_starts_is_that_same_run(
|
||||
"channel_backfill@UC_x",
|
||||
"channel_backfill@UC_y",
|
||||
]
|
||||
# Queued, so its target's newest run row is not this run's yet: a page watching it can tell.
|
||||
assert scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_x")
|
||||
assert not scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_z")
|
||||
live_scheduler.remove_all_jobs() # never run: a run finishing after the test would land in the next
|
||||
|
||||
|
||||
def test_a_request_made_while_its_run_is_under_way_is_the_run_after_it(
|
||||
db, make_user, live_scheduler
|
||||
):
|
||||
"""The scheduler drops a second fire of an id whose run has not returned: the second Takeout file
|
||||
of a pair, uploaded while the first one's import ran, was never imported, and its key stayed
|
||||
queued until a restart (the page spun on "Processing…" for good)."""
|
||||
user = make_user("again@example.com")
|
||||
first_runs, release = threading.Event(), threading.Event()
|
||||
calls = []
|
||||
|
||||
def first():
|
||||
first_runs.set()
|
||||
assert release.wait(10)
|
||||
calls.append("first")
|
||||
return {"n": 1}
|
||||
|
||||
def second():
|
||||
calls.append("second")
|
||||
return {"n": 2}
|
||||
|
||||
scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, first, actor_id=user.id, target="UC_again")
|
||||
assert first_runs.wait(10)
|
||||
scheduler.run_ad_hoc(scheduler.CHANNEL_BACKFILL, second, actor_id=user.id, target="UC_again")
|
||||
assert scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_again")
|
||||
release.set()
|
||||
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline and len(_finished(db, "channel_backfill")) < 2:
|
||||
time.sleep(0.05)
|
||||
assert calls == ["first", "second"]
|
||||
assert sorted(r.result["n"] for r in _finished(db, "channel_backfill")) == [1, 2]
|
||||
assert not scheduler.ad_hoc_queued(scheduler.CHANNEL_BACKFILL, "UC_again")
|
||||
|
||||
|
||||
def test_a_manual_run_leaves_no_actor_behind_on_its_thread(db, make_user):
|
||||
"""The scheduler's pool threads are reused: an actor a manual run left set made the next
|
||||
scheduled run on that thread an admin's "manual" run, with a notice in their inbox."""
|
||||
@@ -475,7 +536,7 @@ def test_a_run_whose_outcome_cannot_be_written_is_not_called_failed(db, make_use
|
||||
scheduler._ad_hoc("channel_backfill", lambda: {"videos_added": 1}, user.id, "UC_blip")
|
||||
assert calls == ["ok"]
|
||||
# Its row is still `running`, but nothing here is: the page does not call it going.
|
||||
(row,) = scheduler.scheduler_snapshot(db)["ad_hoc"]
|
||||
row = _ad_hoc_row(db, "channel_backfill")
|
||||
assert (row["running"], row["status"], row["recent"][0]["status"]) == (
|
||||
False,
|
||||
"interrupted",
|
||||
@@ -529,7 +590,7 @@ def test_an_ad_hoc_run_still_going_shows_although_a_newer_one_ended(db, make_use
|
||||
return {}
|
||||
|
||||
scheduler._ad_hoc("channel_backfill", long_one, user.id, "UC_long")
|
||||
(row,) = rows
|
||||
(row,) = [r for r in rows if r["id"] == "channel_backfill"]
|
||||
assert (row["running"], row["target"], row["target_title"]) == (True, "UC_long", "Long one")
|
||||
# The outcome beside it is the long one's (none yet), never the short one's skip.
|
||||
assert (row["status"], row["result"], row["last_error"]) == (None, None, None)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -44,6 +44,9 @@ UNIT_MODELS = {
|
||||
"PlaylistItem",
|
||||
"SavedView",
|
||||
"ComfortPin",
|
||||
"TakeoutItem",
|
||||
"TakeoutChannelDismissal",
|
||||
"VideoUnmark",
|
||||
"Tag",
|
||||
"ChannelTag",
|
||||
},
|
||||
@@ -360,6 +363,13 @@ UNIT_COMMITS = {
|
||||
"sync/playlists.py:pull_user_playlists",
|
||||
"sync/playlists.py:merge_playlist",
|
||||
"sync/playlists.py:repull_playlist",
|
||||
# The Takeout import is a recorded run with its own session: the watched part lands before
|
||||
# the Watch later part asks YouTube (which commits the videos it fetched).
|
||||
"reader/takeout.py:run_import",
|
||||
# Run inside the Shorts job, on its session: each user's waiting rows land on their own.
|
||||
"reader/takeout.py:apply_waiting",
|
||||
# Inside either of those: the lengths YouTube gave land before the channels are asked for.
|
||||
"reader/takeout.py:_channel_list",
|
||||
},
|
||||
# The catalog's ingest jobs commit per page or batch, a YouTube call between each: a page
|
||||
# fetched and stored is not lost when the next call fails, and an enrich that fails rolls back
|
||||
|
||||
@@ -0,0 +1,538 @@
|
||||
"""The channels a user's Takeout history shows them watching but not following: which channels make
|
||||
the list (five videos of five minutes or more, none they follow, block, put out or own), what the
|
||||
import asks YouTube for it and when, and the channel's watched videos made ready to play."""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app import quota
|
||||
from app.catalog import service as catalog_service
|
||||
from app.catalog.models import BlockedChannel, Channel, ExploredChannel, Subscription, Video
|
||||
from app.db import SessionLocal
|
||||
from app.reader import queries, service, takeout
|
||||
from app.reader.models import TakeoutChannelDismissal, TakeoutItem, TakeoutKind
|
||||
from app.routes import feed as feed_routes
|
||||
from app.sync import explore as sync_explore
|
||||
from app.sync import videos as sync_videos
|
||||
from app.youtube.client import YouTubeError
|
||||
|
||||
T0 = datetime(2026, 5, 1, 12, 0, tzinfo=timezone.utc)
|
||||
LONG = 600
|
||||
SHORT = 120
|
||||
|
||||
|
||||
def _ch(n: int) -> str:
|
||||
return f"UC{n:022d}"
|
||||
|
||||
|
||||
def _vid(channel: int, n: int) -> str:
|
||||
return f"c{channel:02d}v{n:07d}"
|
||||
|
||||
|
||||
def _watched(db, user, channel: int, lengths: list[int | None], *, at=T0, asked=True):
|
||||
"""Stage `len(lengths)` watched videos of a channel, the newest first, each with its length
|
||||
already asked (None: YouTube no longer has it) — or not asked yet."""
|
||||
service.stage_takeout(
|
||||
db,
|
||||
user,
|
||||
TakeoutKind.WATCHED,
|
||||
{_vid(channel, i): (at - timedelta(days=i), _ch(channel)) for i in range(len(lengths))},
|
||||
)
|
||||
db.flush()
|
||||
if asked:
|
||||
ids = [_vid(channel, i) for i in range(len(lengths))]
|
||||
service.record_takeout_lengths(
|
||||
db, user, ids, {v: s for v, s in zip(ids, lengths, strict=True) if s is not None}
|
||||
)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _channel(db, n: int, title: str | None = None):
|
||||
db.add(Channel(id=_ch(n), title=title or f"Channel {n}", details_synced_at=T0))
|
||||
db.commit()
|
||||
|
||||
|
||||
def _listed(db, user) -> list[tuple[str, int]]:
|
||||
return [(ch.id, n) for ch, n, _ in queries.watched_channels(db, user)]
|
||||
|
||||
|
||||
# --- which channels make the list --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_a_channel_makes_the_list_with_five_watched_videos_of_five_minutes_or_more(db, make_user):
|
||||
user = make_user("list@example.com")
|
||||
for n in (1, 2, 3):
|
||||
_channel(db, n)
|
||||
_watched(db, user, 1, [LONG] * 5 + [SHORT] * 3) # five long, three short: on it with 5
|
||||
_watched(db, user, 2, [LONG] * 4 + [SHORT] * 6) # ten watched, only four long: off it
|
||||
_watched(db, user, 3, [299, 300, 300, 300, 300, 300]) # five at exactly five minutes: on it
|
||||
# A tie: channel 1's counted videos were watched later (channel 3's newest is under five minutes).
|
||||
assert _listed(db, user) == [(_ch(1), 5), (_ch(3), 5)]
|
||||
|
||||
|
||||
def test_a_video_youtube_no_longer_has_does_not_count(db, make_user):
|
||||
user = make_user("gone@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 4 + [None] * 3)
|
||||
assert _listed(db, user) == []
|
||||
|
||||
|
||||
def test_the_most_watched_lead_and_a_tie_goes_to_the_one_watched_more_lately(db, make_user):
|
||||
user = make_user("order@example.com")
|
||||
for n in (1, 2, 3):
|
||||
_channel(db, n)
|
||||
_watched(db, user, 1, [LONG] * 6, at=T0 - timedelta(days=100))
|
||||
_watched(db, user, 2, [LONG] * 9, at=T0 - timedelta(days=300))
|
||||
_watched(db, user, 3, [LONG] * 6, at=T0)
|
||||
assert _listed(db, user) == [(_ch(2), 9), (_ch(3), 6), (_ch(1), 6)]
|
||||
last = {ch.id: when for ch, _, when in queries.watched_channels(db, user)}
|
||||
assert last[_ch(3)] == T0 # the last watched of the counted ones
|
||||
|
||||
|
||||
def test_a_channel_the_user_follows_blocked_put_out_or_owns_is_not_on_the_list(db, make_user):
|
||||
user = make_user("off@example.com", yt_channel_id=_ch(4))
|
||||
for n in (1, 2, 3, 4, 5):
|
||||
_channel(db, n)
|
||||
_watched(db, user, n, [LONG] * 5)
|
||||
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(2)))
|
||||
db.commit()
|
||||
service.dismiss_watched_channel(db, user, _ch(3))
|
||||
db.commit()
|
||||
assert _listed(db, user) == [(_ch(5), 5)]
|
||||
|
||||
|
||||
def test_one_users_follows_and_dismissals_leave_anothers_list_alone(db, make_user):
|
||||
me, other = make_user("me@example.com"), make_user("other@example.com")
|
||||
_channel(db, 1)
|
||||
for user in (me, other):
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
db.add(Subscription(user_id=other.id, channel_id=_ch(1)))
|
||||
db.commit()
|
||||
service.dismiss_watched_channel(db, other, _ch(1))
|
||||
db.commit()
|
||||
assert _listed(db, me) == [(_ch(1), 5)]
|
||||
assert _listed(db, other) == []
|
||||
|
||||
|
||||
def test_a_channel_whose_lengths_are_not_all_known_is_pending_until_they_are(db, make_user):
|
||||
user = make_user("pending@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 6, asked=False)
|
||||
_watched(db, user, 2, [LONG] * 4, asked=False) # four watched: it can never make the list
|
||||
assert queries.watched_channels_pending(db, user) == 1
|
||||
assert _listed(db, user) == []
|
||||
ids = [_vid(1, i) for i in range(6)]
|
||||
service.record_takeout_lengths(db, user, ids, dict.fromkeys(ids, LONG))
|
||||
db.commit()
|
||||
assert queries.watched_channels_pending(db, user) == 0
|
||||
assert _listed(db, user) == [(_ch(1), 6)]
|
||||
|
||||
|
||||
def test_a_new_upload_of_the_history_keeps_the_lengths_already_asked(db, make_user):
|
||||
user = make_user("again@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
_watched(db, user, 1, [LONG] * 5, at=T0 + timedelta(days=1), asked=False)
|
||||
assert _listed(db, user) == [(_ch(1), 5)]
|
||||
assert queries.takeout_lengths_to_ask(db, user) == []
|
||||
|
||||
|
||||
# --- what the import asks YouTube for ----------------------------------------------------------
|
||||
|
||||
|
||||
class FakeYouTube:
|
||||
"""Stands in for `YouTubeClient`: answers `videos.list` with the lengths it was given and
|
||||
`channels.list` with a title per channel, and records what it was asked."""
|
||||
|
||||
def __init__(self, lengths: dict[str, int] | None = None, *, fail: bool = False):
|
||||
self.lengths = lengths or {}
|
||||
self.fail = fail
|
||||
self.fail_channels = False
|
||||
self.videos_asked: list[list[str]] = []
|
||||
self.channels_asked: list[list[str]] = []
|
||||
self.actions: list[tuple[int | None, str]] = []
|
||||
|
||||
def __call__(self, db, user):
|
||||
return self
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def get_videos(self, ids):
|
||||
self.videos_asked.append(list(ids))
|
||||
self.actions.append((quota._actor_id.get(), quota._action.get()))
|
||||
if self.fail:
|
||||
raise YouTubeError("refused", reason="quotaExceeded", path="videos")
|
||||
return [
|
||||
{"id": v, "contentDetails": {"duration": f"PT{self.lengths[v]}S"}}
|
||||
for v in ids
|
||||
if v in self.lengths
|
||||
]
|
||||
|
||||
def get_channels(self, ids):
|
||||
self.channels_asked.append(list(ids))
|
||||
self.actions.append((quota._actor_id.get(), quota._action.get()))
|
||||
if self.fail_channels:
|
||||
raise YouTubeError("refused", reason="backendError", path="channels")
|
||||
return [{"id": c, "snippet": {"title": f"Named {c[-2:]}"}} for c in ids]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def youtube(monkeypatch, db, make_user):
|
||||
fetcher = make_user("fetcher@example.com")
|
||||
fake = FakeYouTube()
|
||||
monkeypatch.setattr(takeout, "YouTubeClient", fake)
|
||||
monkeypatch.setattr(takeout, "get_service_user", lambda session: fetcher)
|
||||
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: False)
|
||||
return fake
|
||||
|
||||
|
||||
def test_the_import_asks_the_lengths_the_list_needs_and_names_its_channels(db, make_user, youtube):
|
||||
user = make_user("import@example.com")
|
||||
_watched(db, user, 1, [None] * 7, asked=False) # the most watched channel
|
||||
_watched(db, user, 2, [None] * 5, asked=False)
|
||||
_watched(db, user, 3, [None] * 4, asked=False) # too few to ever make it: not asked about
|
||||
youtube.lengths = {_vid(1, i): LONG for i in range(6)} | {_vid(2, i): SHORT for i in range(5)}
|
||||
|
||||
result = takeout.run_import(user.id)
|
||||
|
||||
asked = [_vid(1, i) for i in range(7)] + [_vid(2, i) for i in range(5)]
|
||||
assert youtube.videos_asked == [asked] # the top channel first
|
||||
assert result["lengths_asked"] == 12
|
||||
# Only channel 1 made it (six long; its seventh is gone, channel 2's are short): it is named.
|
||||
assert youtube.channels_asked == [[_ch(1)]]
|
||||
assert result["channels_described"] == 1
|
||||
assert {
|
||||
(user.id, quota.QuotaAction.VIDEOS_LOOKUP),
|
||||
(user.id, quota.QuotaAction.CHANNELS_DISCOVER),
|
||||
} == set(youtube.actions)
|
||||
db.expire_all()
|
||||
assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 6)]
|
||||
assert queries.watched_channels_pending(db, user) == 0
|
||||
# Asked once: a second run asks nothing again, the gone video included.
|
||||
youtube.videos_asked.clear()
|
||||
youtube.channels_asked.clear()
|
||||
assert "lengths_asked" not in takeout.run_import(user.id)
|
||||
assert youtube.videos_asked == [] and youtube.channels_asked == []
|
||||
|
||||
|
||||
def test_the_lookups_stop_at_the_users_daily_share_and_the_rest_wait(
|
||||
db, make_user, youtube, monkeypatch
|
||||
):
|
||||
user = make_user("share@example.com")
|
||||
_watched(db, user, 1, [None] * 8, asked=False)
|
||||
_watched(db, user, 2, [None] * 6, asked=False)
|
||||
monkeypatch.setattr(takeout, "_lengths", takeout.RateLimiter(10, 3600))
|
||||
takeout.run_import(user.id)
|
||||
assert youtube.videos_asked == [[_vid(1, i) for i in range(8)] + [_vid(2, 0), _vid(2, 1)]]
|
||||
db.expire_all()
|
||||
assert queries.watched_channels_pending(db, user) == 1 # channel 2 still has some to ask
|
||||
youtube.videos_asked.clear()
|
||||
takeout.run_import(user.id)
|
||||
assert youtube.videos_asked == [] # the share is spent for today
|
||||
|
||||
|
||||
@pytest.mark.parametrize("why", ["no_fetcher", "reserve"])
|
||||
def test_nothing_is_asked_without_a_connected_account_or_past_the_reserve(
|
||||
db, make_user, youtube, monkeypatch, why
|
||||
):
|
||||
user = make_user(f"{why}@example.com")
|
||||
_watched(db, user, 1, [None] * 5, asked=False)
|
||||
if why == "no_fetcher":
|
||||
monkeypatch.setattr(takeout, "get_service_user", lambda session: None)
|
||||
else:
|
||||
monkeypatch.setattr(takeout.quota, "backfill_reserve_reached", lambda: True)
|
||||
assert "lengths_asked" not in takeout.run_import(user.id)
|
||||
assert youtube.videos_asked == []
|
||||
assert queries.watched_channels_pending(db, user) == 1
|
||||
|
||||
|
||||
def test_a_youtube_failure_asking_lengths_fails_neither_the_import_nor_its_marks(
|
||||
db, make_user, youtube
|
||||
):
|
||||
user = make_user("fail@example.com")
|
||||
_watched(db, user, 1, [None] * 5, asked=False)
|
||||
youtube.fail = True
|
||||
result = takeout.run_import(user.id)
|
||||
assert "lengths_asked" not in result
|
||||
assert queries.takeout_lengths_to_ask(db, user) == [_vid(1, i) for i in range(5)]
|
||||
db.expire_all()
|
||||
rows = db.execute(select(TakeoutItem).where(TakeoutItem.user_id == user.id)).scalars()
|
||||
assert all(r.looked_up_at is None for r in rows) # left to ask again
|
||||
|
||||
|
||||
def test_the_shorts_job_asks_what_an_import_could_not(db, make_user, youtube):
|
||||
user = make_user("later@example.com")
|
||||
_watched(db, user, 1, [None] * 5, asked=False)
|
||||
youtube.lengths = {_vid(1, i): LONG for i in range(5)}
|
||||
assert takeout.apply_waiting(db)["takeout_lengths_asked"] == 5
|
||||
db.expire_all()
|
||||
assert [n for _, n, _ in queries.watched_channels(db, user)] == [5]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("why", ["follows", "blocked", "dismissed", "own"])
|
||||
def test_a_channel_off_the_list_is_not_asked_about_nor_looked_at_again(
|
||||
db, make_user, youtube, monkeypatch, why
|
||||
):
|
||||
user = make_user(f"off-{why}@example.com", yt_channel_id=_ch(1) if why == "own" else None)
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [None] * 5, asked=False)
|
||||
if why == "follows":
|
||||
db.add(Subscription(user_id=user.id, channel_id=_ch(1)))
|
||||
elif why == "blocked":
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
|
||||
elif why == "dismissed":
|
||||
service.dismiss_watched_channel(db, user, _ch(1))
|
||||
db.commit()
|
||||
runs = []
|
||||
monkeypatch.setattr(takeout, "_channel_list", lambda session, u: runs.append(u.id) or {})
|
||||
assert takeout.apply_waiting(db) == {}
|
||||
# Its rows are never asked about, so they would stay "to ask" for good: the Shorts job must not
|
||||
# take the user up again for them on every run.
|
||||
assert runs == []
|
||||
assert youtube.videos_asked == []
|
||||
|
||||
|
||||
def test_naming_the_channels_is_tried_again_and_meanwhile_the_list_says_it_is_not_complete(
|
||||
db, make_user, youtube
|
||||
):
|
||||
user = make_user("unnamed@example.com")
|
||||
_watched(db, user, 1, [None] * 5, asked=False)
|
||||
youtube.lengths = {_vid(1, i): LONG for i in range(5)}
|
||||
youtube.fail_channels = True
|
||||
takeout.run_import(user.id)
|
||||
db.expire_all()
|
||||
# Every length is known, the channel could not be named: not on the list, and not complete.
|
||||
assert queries.takeout_lengths_to_ask(db, user) == []
|
||||
assert _listed(db, user) == []
|
||||
assert queries.watched_channels_pending(db, user) == 1
|
||||
youtube.fail_channels = False
|
||||
takeout.apply_waiting(db)
|
||||
db.expire_all()
|
||||
assert [(ch.title, n) for ch, n, _ in queries.watched_channels(db, user)] == [("Named 01", 5)]
|
||||
assert queries.watched_channels_pending(db, user) == 0
|
||||
|
||||
|
||||
def test_a_listed_channel_still_to_be_described_is_not_counted_as_one_to_come(db, make_user):
|
||||
user = make_user("stub@example.com")
|
||||
# A stub the catalog made when one of its videos came in: a title, no details yet.
|
||||
db.add(Channel(id=_ch(1), title="From a video"))
|
||||
db.commit()
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
assert queries.watched_channels_to_describe(db, user) == [_ch(1)]
|
||||
assert _listed(db, user) == [(_ch(1), 5)]
|
||||
# It is on the list already: "the list may grow" would promise one more that cannot come.
|
||||
assert queries.watched_channels_pending(db, user) == 0
|
||||
|
||||
|
||||
def test_the_lengths_leave_the_watch_later_lookups_their_own_share(
|
||||
db, make_user, youtube, monkeypatch
|
||||
):
|
||||
user = make_user("shares@example.com")
|
||||
monkeypatch.setattr(takeout, "_lookups", takeout.RateLimiter(10, 3600))
|
||||
monkeypatch.setattr(takeout, "_lengths", takeout.RateLimiter(10, 3600))
|
||||
_watched(db, user, 1, [None] * 15, asked=False)
|
||||
takeout.run_import(user.id) # the history: its lengths spend all they may today
|
||||
youtube.videos_asked.clear()
|
||||
service.stage_takeout(db, user, TakeoutKind.WATCH_LATER, {"later000001": (T0, None)})
|
||||
db.commit()
|
||||
takeout.run_import(user.id) # then the export's Watch later file
|
||||
assert ["later000001"] in youtube.videos_asked
|
||||
|
||||
|
||||
# --- the channel's watched videos, ready to play ------------------------------------------------
|
||||
|
||||
|
||||
def _fake_ensure(calls):
|
||||
def ensure(session, yt, ids, *, explore=False):
|
||||
calls.append((list(ids), explore))
|
||||
for vid in ids:
|
||||
if vid.endswith("4"):
|
||||
continue # gone from YouTube since its length was asked
|
||||
session.add(Video(id=vid, channel_id=_ch(1), published_at=T0, via_explore=explore))
|
||||
session.commit()
|
||||
|
||||
return ensure
|
||||
|
||||
|
||||
def test_a_channels_watched_videos_come_ready_to_play_and_explored(
|
||||
db, make_user, as_user, monkeypatch
|
||||
):
|
||||
user = make_user("play@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG, SHORT, LONG, None, LONG, LONG, LONG])
|
||||
db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0)) # already in the catalog
|
||||
db.commit()
|
||||
calls = []
|
||||
monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
|
||||
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
|
||||
|
||||
resp = as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos")
|
||||
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
# Only the counted ones, newest first; the short one and the gone ones are left out.
|
||||
assert calls == [([_vid(1, 2), _vid(1, 4), _vid(1, 5), _vid(1, 6)], True)]
|
||||
assert [it["id"] for it in body["items"]] == [_vid(1, 0), _vid(1, 2), _vid(1, 5), _vid(1, 6)]
|
||||
assert body["items"][0]["takeout_watched_at"].startswith("2026-05-01")
|
||||
assert body["gone"] == 2 # one YouTube had lost before, one since
|
||||
assert body["ok"] is True
|
||||
db.expire_all()
|
||||
assert db.execute(
|
||||
select(ExploredChannel).where(
|
||||
ExploredChannel.user_id == user.id, ExploredChannel.channel_id == _ch(1)
|
||||
)
|
||||
).scalar_one()
|
||||
assert (
|
||||
db.get(Channel, _ch(1)).from_explore is True
|
||||
) # nobody follows it: the cleanup's to reclaim
|
||||
|
||||
|
||||
def test_a_followed_channels_watched_videos_come_in_as_the_catalogs_own(
|
||||
db, make_user, as_user, monkeypatch
|
||||
):
|
||||
user, follower = make_user("viewer@example.com"), make_user("follower@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
db.add(Subscription(user_id=follower.id, channel_id=_ch(1)))
|
||||
db.commit()
|
||||
calls = []
|
||||
monkeypatch.setattr(feed_routes, "ensure_videos", _fake_ensure(calls))
|
||||
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
|
||||
assert as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 200
|
||||
# Someone follows it, so it is no exploration's to reclaim: a video flagged as one would never
|
||||
# be cleared again (subscribing clears only an explored channel's) and never marked watched.
|
||||
assert calls == [([_vid(1, i) for i in range(5)], False)]
|
||||
db.expire_all()
|
||||
assert db.get(Channel, _ch(1)).from_explore is False
|
||||
|
||||
|
||||
def test_a_follow_landing_while_the_videos_come_in_leaves_none_of_them_flagged(
|
||||
db, make_user, as_user, monkeypatch
|
||||
):
|
||||
user, follower = make_user("racer@example.com"), make_user("late@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
fake = _fake_ensure(calls := [])
|
||||
|
||||
def ensure(session, yt, ids, *, explore=False):
|
||||
# Someone subscribes, in a request of their own, while YouTube answers this one.
|
||||
with SessionLocal() as other:
|
||||
ch = other.get(Channel, _ch(1))
|
||||
catalog_service.record_subscription(other, follower, ch, "sub-late")
|
||||
other.commit()
|
||||
fake(session, yt, ids, explore=explore)
|
||||
|
||||
monkeypatch.setattr(feed_routes, "ensure_videos", ensure)
|
||||
monkeypatch.setattr(feed_routes.quota, "backfill_reserve_reached", lambda: False)
|
||||
assert as_user(user).post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 200
|
||||
assert calls[0][1] is True # asked as the exploration's
|
||||
db.expire_all()
|
||||
flagged = db.scalars(select(Video.id).where(Video.via_explore.is_(True))).all()
|
||||
assert (db.get(Channel, _ch(1)).from_explore, flagged) == (False, [])
|
||||
|
||||
|
||||
def test_the_cleanup_keeps_a_channel_a_watch_history_lists_and_reclaims_its_videos(db, make_user):
|
||||
user = make_user("kept@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
catalog_service.explore(db, user, db.get(Channel, _ch(1)))
|
||||
db.add(Video(id=_vid(1, 0), channel_id=_ch(1), published_at=T0, via_explore=True))
|
||||
db.add(Channel(id=_ch(2), from_explore=True)) # explored, in nobody's history
|
||||
db.commit()
|
||||
db.execute(update(ExploredChannel).values(created_at=T0 - timedelta(days=365)))
|
||||
db.commit()
|
||||
out = sync_explore.purge_unkept_explored(db)
|
||||
# The unplayed video goes (opening the list's videos brings it back); the channel stays listed.
|
||||
assert (out["videos_deleted"], out["channels_deleted"]) == (1, 1)
|
||||
db.expire_all()
|
||||
assert db.get(Channel, _ch(2)) is None
|
||||
assert _listed(db, user) == [(_ch(1), 5)]
|
||||
|
||||
|
||||
def test_a_blocked_or_unknown_channels_videos_are_refused(db, make_user, as_user):
|
||||
user = make_user("refused@example.com")
|
||||
_channel(db, 1)
|
||||
db.add(BlockedChannel(user_id=user.id, channel_id=_ch(1)))
|
||||
db.commit()
|
||||
client = as_user(user)
|
||||
assert client.post(f"/api/takeout/channels/{_ch(1)}/videos").status_code == 403
|
||||
assert client.post(f"/api/takeout/channels/{_ch(9)}/videos").status_code == 404
|
||||
|
||||
|
||||
def test_ensure_videos_adds_an_explorations_videos_as_its_own(db):
|
||||
db.add(Channel(id=_ch(1), title="C"))
|
||||
db.commit()
|
||||
|
||||
class YT:
|
||||
def get_videos(self, ids):
|
||||
return [{"id": v, "snippet": {"channelId": _ch(1)}} for v in ids]
|
||||
|
||||
sync_videos.ensure_videos(db, YT(), ["exploredvid"], explore=True)
|
||||
sync_videos.ensure_videos(db, YT(), ["organicvid0"])
|
||||
db.expire_all()
|
||||
assert db.get(Video, "exploredvid").via_explore is True
|
||||
assert db.get(Video, "organicvid0").via_explore is False
|
||||
|
||||
|
||||
# --- the routes -----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_the_list_route_says_what_it_holds_and_whether_a_history_was_uploaded(
|
||||
db, make_user, as_user
|
||||
):
|
||||
user = make_user("route@example.com")
|
||||
client = as_user(user)
|
||||
assert client.get("/api/takeout/channels").json() == {
|
||||
"imported": False,
|
||||
"pending": 0,
|
||||
"channels": [],
|
||||
}
|
||||
_channel(db, 1, "Some channel")
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
_watched(db, user, 2, [None] * 5, asked=False)
|
||||
body = client.get("/api/takeout/channels").json()
|
||||
assert body["imported"] is True and body["pending"] == 1
|
||||
assert [(c["id"], c["title"], c["watched"]) for c in body["channels"]] == [
|
||||
(_ch(1), "Some channel", 5)
|
||||
]
|
||||
assert body["channels"][0]["last_watched_at"].startswith("2026-05-01")
|
||||
|
||||
|
||||
def test_a_channel_put_out_of_the_list_comes_back_when_restored(db, make_user, as_user):
|
||||
user = make_user("dismiss@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, user, 1, [LONG] * 5)
|
||||
client = as_user(user)
|
||||
url = f"/api/takeout/channels/{_ch(1)}/dismissal"
|
||||
assert client.put(url).json() == {"ok": True, "channel_id": _ch(1)}
|
||||
assert client.put(url).status_code == 200 # again: no error
|
||||
assert client.get("/api/takeout/channels").json()["channels"] == []
|
||||
assert client.delete(url).json() == {"ok": True, "channel_id": _ch(1)}
|
||||
assert [c["id"] for c in client.get("/api/takeout/channels").json()["channels"]] == [_ch(1)]
|
||||
db.expire_all()
|
||||
assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []
|
||||
|
||||
|
||||
def test_only_a_channel_of_the_users_own_history_can_be_put_out_of_the_list(db, make_user, as_user):
|
||||
user, other = make_user("mine@example.com"), make_user("theirs@example.com")
|
||||
_channel(db, 1)
|
||||
_watched(db, other, 1, [LONG] * 5)
|
||||
client = as_user(user)
|
||||
assert client.put(f"/api/takeout/channels/{_ch(1)}/dismissal").status_code == 404
|
||||
assert client.put(f"/api/takeout/channels/{_ch(9)}/dismissal").status_code == 404
|
||||
db.expire_all()
|
||||
assert db.execute(select(TakeoutChannelDismissal)).scalars().all() == []
|
||||
|
||||
|
||||
def test_a_dismissal_of_an_overlong_channel_id_is_refused_before_the_database(
|
||||
db, make_user, as_user
|
||||
):
|
||||
user = make_user("long@example.com")
|
||||
assert as_user(user).put(f"/api/takeout/channels/{'U' * 33}/dismissal").status_code == 422
|
||||
@@ -1,9 +1,9 @@
|
||||
import { test as setup, expect, type Page } from "@playwright/test";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, readFileSync } from "node:fs";
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { MEMBER_STORAGE } from "./helpers";
|
||||
import { MEMBER_STORAGE, SEED_FILE } from "./helpers";
|
||||
import { retryConnect } from "./retryConnect";
|
||||
|
||||
/**
|
||||
@@ -99,6 +99,8 @@ setup("seed + authenticate", async ({ page, browser }) => {
|
||||
{ input: readFileSync(SEED_SCRIPT) },
|
||||
);
|
||||
console.log(`[e2e seed] ${out.toString().trim()}`);
|
||||
// What the seed chose, for the specs that need one of its videos (helpers.seeded).
|
||||
writeFileSync(SEED_FILE, out);
|
||||
} catch (err: unknown) {
|
||||
const e = err as { stderr?: Buffer; message?: string };
|
||||
throw new Error(`E2E seed failed: ${e.stderr?.toString() || e.message}`);
|
||||
|
||||
@@ -1,8 +1,18 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { expect, type Page } from "@playwright/test";
|
||||
|
||||
/** The plain member's saved session (global.setup.ts signs it in); `member.spec.ts` and
|
||||
* `devices.spec.ts` run from it. */
|
||||
export const MEMBER_STORAGE = "e2e/.auth/member.json";
|
||||
// The seed's own answer (scripts/e2e_seed.py), written by global.setup: the videos it chose.
|
||||
export const SEED_FILE = "e2e/.auth/seed.json";
|
||||
|
||||
/** The videos the seed chose for the main account under `key` in its answer. */
|
||||
export function seeded(key: string): string[] {
|
||||
const ids = (JSON.parse(readFileSync(SEED_FILE, "utf-8")) as Record<string, unknown>)[key];
|
||||
if (!Array.isArray(ids) || !ids.length) throw new Error(`the seed chose no ${key} videos`);
|
||||
return ids as string[];
|
||||
}
|
||||
|
||||
/** Open the app and wait for the authenticated shell (the nav rail) to be ready. */
|
||||
export async function openApp(page: Page, path = "/"): Promise<void> {
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
import { test, expect } from "@playwright/test";
|
||||
import { openApp, gotoPage, seeded } from "./helpers";
|
||||
|
||||
// Settings › Account › YouTube history: a Google Takeout watch history, uploaded through the real
|
||||
// picker, is read, imported in the background, and the section says what it did — a video the
|
||||
// catalog knows is marked watched, one it does not know is remembered for later. The seed clears
|
||||
// the account's kept Takeout videos each run, but the projects share one account: a later one's
|
||||
// import of the same video adds nothing new, so what each adds is counted against what was there
|
||||
// before it.
|
||||
test("a Takeout watch history uploads, imports, and the section says what it did", async ({
|
||||
page,
|
||||
}) => {
|
||||
await openApp(page);
|
||||
await gotoPage(page, "settings");
|
||||
await page.getByRole("tab", { name: "Account" }).click();
|
||||
await expect(page.getByRole("heading", { name: "YouTube history" })).toBeVisible();
|
||||
// The first project's upload is an import already made: its "New import" opens the drop zone.
|
||||
const zone = page.getByRole("button", { name: "Drop the files here, or choose them" });
|
||||
const again = page.getByRole("button", { name: "New import" });
|
||||
await expect(zone.or(again)).toBeVisible();
|
||||
if (await again.isVisible()) await again.click();
|
||||
await expect(zone).toBeVisible();
|
||||
const before = (await (await page.request.get("/api/takeout")).json()) as {
|
||||
watched?: Record<string, number>;
|
||||
};
|
||||
const watchedBefore = before.watched?.watched ?? 0;
|
||||
// A video the catalog knows and the Shorts probe has cleared, chosen by the seed (one not probed
|
||||
// yet would wait for the probe): the next one no earlier project of this run has imported.
|
||||
const known = seeded("takeout_known")[watchedBefore];
|
||||
expect(known, "the seed chose too few videos for every project").toBeTruthy();
|
||||
|
||||
// Watched now: later than any state the seed set, which the import would otherwise leave alone.
|
||||
const now = new Date().toISOString();
|
||||
const history = [
|
||||
{ titleUrl: `https://www.youtube.com/watch?v=${known}`, time: now },
|
||||
{ titleUrl: "https://www.youtube.com/watch?v=e2eUnknown0", time: now },
|
||||
{
|
||||
titleUrl: "https://www.youtube.com/watch?v=e2eAdvert00",
|
||||
time: now,
|
||||
details: [{ name: "From Google Ads" }],
|
||||
},
|
||||
];
|
||||
const uploaded = page.waitForResponse(
|
||||
(r) => r.url().endsWith("/api/takeout") && r.request().method() === "POST",
|
||||
);
|
||||
await page.locator('input[type="file"][accept=".json,.csv"]').setInputFiles({
|
||||
name: "watch-history.json",
|
||||
mimeType: "application/json",
|
||||
buffer: Buffer.from(JSON.stringify(history)),
|
||||
});
|
||||
expect((await uploaded).ok()).toBeTruthy();
|
||||
|
||||
await expect(page.getByText("watch-history.json: 2 videos read")).toBeVisible();
|
||||
const line = page.getByTestId("takeout-last");
|
||||
await expect(line).toContainText("Last import:");
|
||||
await expect(line).toContainText(`${watchedBefore + 1} watched`);
|
||||
await expect(line).toContainText("1 remembered");
|
||||
await expect(page.getByRole("button", { name: "New import" })).toBeVisible();
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// ui/DropZone: a drop or a pick hands over the files (only the first unless `multiple`), the drag
|
||||
// highlight survives the cursor crossing the zone's own children, and a disabled zone takes nothing.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import { DropZone } from "./DropZone";
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
const files = (...names: string[]) => names.map((n) => new File(["x"], n));
|
||||
function mount(props: Partial<Parameters<typeof DropZone>[0]> = {}) {
|
||||
const onFiles = vi.fn();
|
||||
render(
|
||||
<DropZone onFiles={onFiles} {...props}>
|
||||
{(over) => <span data-testid="inner">{over ? "over" : "idle"}</span>}
|
||||
</DropZone>,
|
||||
);
|
||||
return { onFiles, zone: screen.getByRole("button"), inner: screen.getByTestId("inner") };
|
||||
}
|
||||
|
||||
describe("DropZone", () => {
|
||||
it("a drop hands over the first file, or all of them when multiple", () => {
|
||||
const one = mount();
|
||||
fireEvent.drop(one.zone, { dataTransfer: { files: files("a", "b") } });
|
||||
expect(one.onFiles.mock.calls[0]![0].map((f: File) => f.name)).toEqual(["a"]);
|
||||
cleanup();
|
||||
const many = mount({ multiple: true });
|
||||
fireEvent.drop(many.zone, { dataTransfer: { files: files("a", "b") } });
|
||||
expect(many.onFiles.mock.calls[0]![0].map((f: File) => f.name)).toEqual(["a", "b"]);
|
||||
});
|
||||
|
||||
it("a pick hands over the picked files; an empty pick hands over nothing", () => {
|
||||
const { onFiles } = mount({ multiple: true });
|
||||
const input = document.querySelector<HTMLInputElement>('input[type="file"]')!;
|
||||
expect(input.multiple).toBe(true);
|
||||
fireEvent.change(input, { target: { files: [] } });
|
||||
expect(onFiles).not.toHaveBeenCalled();
|
||||
fireEvent.change(input, { target: { files: files("c") } });
|
||||
expect(onFiles.mock.calls[0]![0].map((f: File) => f.name)).toEqual(["c"]);
|
||||
});
|
||||
|
||||
it("stays highlighted while the cursor crosses its own children, and clears on the last leave", () => {
|
||||
const { zone, inner } = mount();
|
||||
fireEvent.dragEnter(zone);
|
||||
fireEvent.dragEnter(inner); // into a child: the zone gets its leave after this enter
|
||||
fireEvent.dragLeave(zone);
|
||||
expect(inner.textContent).toBe("over");
|
||||
fireEvent.dragLeave(inner);
|
||||
expect(inner.textContent).toBe("idle");
|
||||
});
|
||||
|
||||
it("a disabled zone takes no drop", () => {
|
||||
const { zone, onFiles } = mount({ disabled: true });
|
||||
fireEvent.drop(zone, { dataTransfer: { files: files("a") } });
|
||||
expect(onFiles).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("an invalid zone reads as invalid and points at the reason", () => {
|
||||
const { zone } = mount({ invalid: true, describedBy: "why" });
|
||||
expect(zone.getAttribute("aria-invalid")).toBe("true");
|
||||
expect(zone.getAttribute("aria-describedby")).toBe("why");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
import { useRef, useState, type ReactNode, type Ref } from "react";
|
||||
import clsx from "clsx";
|
||||
import { Pressable } from "./Pressable";
|
||||
|
||||
// A place to drop files onto, which is also the button that opens the native file dialog: the
|
||||
// dashed surface an upload starts from. It owns what each such surface needed and the first one
|
||||
// got right only after a flicker: dragenter/dragleave fire as the cursor crosses the zone's OWN
|
||||
// children (an icon, a line of text), so a plain boolean flickers — enter and leave are counted
|
||||
// and the highlight clears at depth 0. The caller draws the inside, told whether a drag is over it.
|
||||
export function DropZone({
|
||||
ref,
|
||||
onFiles,
|
||||
multiple = false,
|
||||
accept,
|
||||
disabled = false,
|
||||
invalid = false,
|
||||
describedBy,
|
||||
className,
|
||||
children,
|
||||
}: {
|
||||
/** The zone's button, for a caller that moves the focus to it. */
|
||||
ref?: Ref<HTMLButtonElement>;
|
||||
/** The files dropped or picked, never empty. One unless `multiple`. */
|
||||
onFiles: (files: [File, ...File[]]) => void;
|
||||
multiple?: boolean;
|
||||
/** The picker's filter (`<input accept>`); a drop is not filtered by it. */
|
||||
accept?: string;
|
||||
disabled?: boolean;
|
||||
/** What the zone holds is refused: the border turns danger and `describedBy` says why. */
|
||||
invalid?: boolean;
|
||||
describedBy?: string;
|
||||
/** Lays the zone out (its padding); the border and the drag paint are the zone's. */
|
||||
className?: string;
|
||||
children: (dragOver: boolean) => ReactNode;
|
||||
}) {
|
||||
const [dragOver, setDragOver] = useState(false);
|
||||
const depth = useRef(0);
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const take = (list: FileList | null | undefined) => {
|
||||
const [first, ...rest] = [...(list ?? [])];
|
||||
if (first) onFiles(multiple ? [first, ...rest] : [first]);
|
||||
};
|
||||
return (
|
||||
<>
|
||||
<Pressable
|
||||
ref={ref}
|
||||
align="center"
|
||||
onClick={() => inputRef.current?.click()}
|
||||
onDragEnter={(e) => {
|
||||
e.preventDefault();
|
||||
depth.current += 1;
|
||||
setDragOver(true);
|
||||
}}
|
||||
onDragOver={(e) => e.preventDefault()}
|
||||
onDragLeave={() => {
|
||||
depth.current = Math.max(0, depth.current - 1);
|
||||
if (depth.current === 0) setDragOver(false);
|
||||
}}
|
||||
onDrop={(e) => {
|
||||
e.preventDefault();
|
||||
depth.current = 0;
|
||||
setDragOver(false);
|
||||
if (!disabled) take(e.dataTransfer.files);
|
||||
}}
|
||||
disabled={disabled}
|
||||
// The refusal is tied to the control it invalidates, not left to a one-shot announcement:
|
||||
// the zone reads as invalid and points at the sentence, so a reader who tabs back to it
|
||||
// hears WHY. (Measured, because `aria-invalid` is not in `button`'s ARIA-supported list:
|
||||
// Chromium's accessibility node for a native button does carry `invalid` from it.) The
|
||||
// signal that is NOT colour is the caller's sentence — the border only reinforces it.
|
||||
aria-invalid={invalid}
|
||||
aria-describedby={describedBy}
|
||||
// `dragOver` outranks `invalid`: a drag is transient and the refusal repaints the instant
|
||||
// the drop lands, whereas a zone that stays red under the pointer tells the user their drop
|
||||
// target is dead — in the one state where they are trying to replace the file.
|
||||
className={clsx(
|
||||
"w-full flex flex-col items-center justify-center gap-2 rounded-xl border-2 border-dashed transition disabled:opacity-60",
|
||||
dragOver
|
||||
? "border-accent bg-accent/5"
|
||||
: invalid
|
||||
? "border-danger"
|
||||
: "border-border hover:border-fg/40",
|
||||
className,
|
||||
)}
|
||||
>
|
||||
{children(dragOver)}
|
||||
</Pressable>
|
||||
<input
|
||||
ref={inputRef}
|
||||
type="file"
|
||||
className="hidden"
|
||||
multiple={multiple}
|
||||
accept={accept}
|
||||
onChange={(e) => {
|
||||
take(e.target.files);
|
||||
e.target.value = ""; // let re-picking the same file fire onChange again
|
||||
}}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -35,13 +35,13 @@ Counter-examples, deliberately NOT here: `PosterTile` (knows about Plex montage
|
||||
|
||||
## What lives here
|
||||
|
||||
| area | components |
|
||||
| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| overlays & anchoring | `AnchoredPopover` + `anchoredPopoverHooks` (the popover engine: arrows, typeahead, Escape, anchor-flip), `ActionMenu`, `MembershipMenu` (the body of a "which sets is it in" menu: checkbox rows + a field naming a new set), `ContextMenu` + `useContextMenu` (the SAME menu, opened by right-click / long-press / Shift+F10 / a row's `MoreButton` "…" via `buttonProps`; `anchor` for a panel a row opens; `PageMenu` for a page's background menu), `Modal` (+ `ModalFooter`; `initialFocus` by purpose, C4), `Overlay`, `SidePanel`, `Tooltip`, `Toaster` (+ `toastLevels`, `overlayOrder`; `lib/toastDock` docks one control under the stack, in every host), `CommandPalette`, `KeyboardHelp`, `ConfirmProvider` + `confirmContext` (`useConfirm` / `useChoice`: the app's promise-based confirm, mounted once at the root; a decision with more than two outcomes is one `useChoice`, never two chained confirms), `dialogFocus` (the focus half of the dialog contract: `Modal`'s, and the wizard's own shell), `menuGroups` (one menu from its groups, a rule only between non-empty ones), `pageMenuSlot` + `pageMenuItems` (a page's background menu, mirroring its own toolbar) |
|
||||
| actions & inputs | `Button`, `IconButton`, `Pressable` (`selectable` when its text is a title or a name), `Chip` + `chipStyles`, `DragHandle` + `OrderButtons` / `useOrderItems` (the non-drag reorder, WCAG 2.5.7) (one per button family, docs/DESIGN.md; a raw `<button>` or a bare `FocusSafeButton` outside the control primitives — eslint `BUTTON_PRIMITIVES` — is a lint error), `Input` + `fieldStyles`, `RenameField` (a name typed in place of the name: Enter or blur saves, Escape cancels; the caller hands the keyboard back), `Meter` (how full something is against a limit — one danger line for every place showing the same measure), `TimecodeField` + `useTimecodeField` / `TimecodeError` (a cut's time typed in: the draft, held by the caller, until Enter or blur takes it, then the value as it landed; unreadable stays once offered, marked invalid, and the caller places the line saying how to type one in its flow, outside a label; a draft goes when the value moves under it), `NativeControls`, `MenuCheck` (the drawn check of a menu row or a whole-row checkbox button; its tick is the `chosen` colour, which `media-marks` turns amber on the player chrome), `mediaChrome` (`MEDIA_CHOSEN`, a chosen tab or toggle on the chrome), `SelectTrigger` (the field family; a raw `<input>` is a lint error), `useReturnFocus`, `surfaces` (the B5 glass roles; a bare glass class is a lint error), `Badge` (a static label), `LinkButton` / `IconLink` (a link wearing the button face) over `buttonStyles`, `Skeleton` (B8 first loads, through `LoadingState shape`), `NavRail` (B7; `NavRailGroup` folds a set of rare pages into one row with a panel), `form`, `DraftSaveBar`, `Combobox`, `FilterSelect`, `SegmentedToggle`, `SortControl`, `ColumnSortControl`, `PriorityStepper`, `UndoToolbar`, `ViewSwitcher` (a list's display modes behind one toolbar icon), `copyItems` (`useCopyItems`: the "Copy title" / "Copy link" rows every object's menu ends with) + `useCopyFeedback` (a Copy button's "Copied" moment), `MatchToggle` (a multi-value filter's any/all switch), `LanguageSwitcher` (presentational: the caller decides what a change does), `sortableA11y` (what a screen reader hears during a keyboard reorder) |
|
||||
| data display | `DataTable` (+ `progressiveRows`: a long page mounts in steps), `tableHeader` + `tableAlign`, `HiddenFilterCue` (names a filter whose column is off screen), `Pager` + `pagerContent`, `VirtualGrid` + `gridColumns` (the column/width maths, incl. the ultrawide `maxCols` cap), `readAhead`, `Carousel`, `MediaCard` + `mediaCardStyles` (the one media card, docs/DESIGN.md "Media card") + `MediaPreview` / `useMediaPreview` (its mouse-rest preview), `NavGrid` (a plain grid, list or table body the keyboard walks — `DataTable`, `VirtualGrid` and `Carousel` take `keyboardNav`), `RemoteImage` (every network picture: no Referer, retries past the negative cache, a fallback instead of the broken-image mark — a bare `<img>` is linted out), `Avatar`, `CountBadge`, `ItemCount`, `FilterChipGroup`, `RelativeTime` ("3d ago" with the exact minute on hover, docs/DESIGN.md "Copy"), `ClampedText` (a long text clamped to a few lines; More / Less only when it is cut), `StatTiles` + `StatTile` / `StatBars` (number tiles, a breakdown as horizontal bars), `RangeTrack` + `trackPosition` + `RulerTicks` (a time track over a window of a media item: regions, slider ends the arrows step, a playhead, a background slot — the editor's cuts, the players' clip panel), `CropFrame` (a crop frame over a picture, on the picture and not its bars: drag, edges, corners, the arrows; the editor's crop, the players' clip crop), `reveal` (a card's hover chrome: hover or keyboard focus on a mouse, never on touch) |
|
||||
| page & panel structure | `PageShell` + `pageToolbarSlot` + `pageHeaderSlot` (`PageHeader`: a page fills the floating header) + `pageHeading`, `PageScroller`, `Headings` (`PageHeading`, `SectionLabel`), `SkipLink` + `landmarks`, `FilterFold` (B4), `scrollFold` (`useScrollFold`: a fixed header that folds once its content scrolls), `PanelGroup`, `PanelGroups`, `Tabs` + `tabPanel`, `BackToTop` + `backToTopState` (PageScroller keeps the page's end clear of it), `FilterRail` (the filter-rail shell a rail fills with its groups: title bar, customize, reset), `stickyTop` (a sticky header at the scroller's top turns its top fade off) |
|
||||
| state & messaging | `QueryState`, `Banner`, `InlineNotice`, `CrashBoundary` |
|
||||
| area | components |
|
||||
| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| overlays & anchoring | `AnchoredPopover` + `anchoredPopoverHooks` (the popover engine: arrows, typeahead, Escape, anchor-flip), `ActionMenu`, `MembershipMenu` (the body of a "which sets is it in" menu: checkbox rows + a field naming a new set), `ContextMenu` + `useContextMenu` (the SAME menu, opened by right-click / long-press / Shift+F10 / a row's `MoreButton` "…" via `buttonProps`; `anchor` for a panel a row opens; `PageMenu` for a page's background menu), `Modal` (+ `ModalFooter`; `initialFocus` by purpose, C4), `Overlay`, `SidePanel`, `Tooltip`, `Toaster` (+ `toastLevels`, `overlayOrder`; `lib/toastDock` docks one control under the stack, in every host), `CommandPalette`, `KeyboardHelp`, `ConfirmProvider` + `confirmContext` (`useConfirm` / `useChoice`: the app's promise-based confirm, mounted once at the root; a decision with more than two outcomes is one `useChoice`, never two chained confirms), `dialogFocus` (the focus half of the dialog contract: `Modal`'s, and the wizard's own shell), `menuGroups` (one menu from its groups, a rule only between non-empty ones), `pageMenuSlot` + `pageMenuItems` (a page's background menu, mirroring its own toolbar) |
|
||||
| actions & inputs | `Button`, `IconButton`, `Pressable` (`selectable` when its text is a title or a name), `Chip` + `chipStyles`, `DragHandle` + `OrderButtons` / `useOrderItems` (the non-drag reorder, WCAG 2.5.7) (one per button family, docs/DESIGN.md; a raw `<button>` or a bare `FocusSafeButton` outside the control primitives — eslint `BUTTON_PRIMITIVES` — is a lint error), `Input` + `fieldStyles`, `RenameField` (a name typed in place of the name: Enter or blur saves, Escape cancels; the caller hands the keyboard back), `DropZone` (the dashed place a file upload starts from: a drop, or a click that opens the file dialog), `Meter` (how full something is against a limit — one danger line for every place showing the same measure), `TimecodeField` + `useTimecodeField` / `TimecodeError` (a cut's time typed in: the draft, held by the caller, until Enter or blur takes it, then the value as it landed; unreadable stays once offered, marked invalid, and the caller places the line saying how to type one in its flow, outside a label; a draft goes when the value moves under it), `NativeControls`, `MenuCheck` (the drawn check of a menu row or a whole-row checkbox button; its tick is the `chosen` colour, which `media-marks` turns amber on the player chrome), `mediaChrome` (`MEDIA_CHOSEN`, a chosen tab or toggle on the chrome), `SelectTrigger` (the field family; a raw `<input>` is a lint error), `useReturnFocus`, `surfaces` (the B5 glass roles; a bare glass class is a lint error), `Badge` (a static label), `LinkButton` / `IconLink` (a link wearing the button face) over `buttonStyles`, `Skeleton` (B8 first loads, through `LoadingState shape`), `NavRail` (B7; `NavRailGroup` folds a set of rare pages into one row with a panel), `form`, `DraftSaveBar`, `Combobox`, `FilterSelect`, `SegmentedToggle`, `SortControl`, `ColumnSortControl`, `PriorityStepper`, `UndoToolbar`, `ViewSwitcher` (a list's display modes behind one toolbar icon), `copyItems` (`useCopyItems`: the "Copy title" / "Copy link" rows every object's menu ends with) + `useCopyFeedback` (a Copy button's "Copied" moment), `MatchToggle` (a multi-value filter's any/all switch), `LanguageSwitcher` (presentational: the caller decides what a change does), `sortableA11y` (what a screen reader hears during a keyboard reorder) |
|
||||
| data display | `DataTable` (+ `progressiveRows`: a long page mounts in steps), `tableHeader` + `tableAlign`, `HiddenFilterCue` (names a filter whose column is off screen), `Pager` + `pagerContent`, `VirtualGrid` + `gridColumns` (the column/width maths, incl. the ultrawide `maxCols` cap), `readAhead`, `Carousel`, `MediaCard` + `mediaCardStyles` (the one media card, docs/DESIGN.md "Media card") + `MediaPreview` / `useMediaPreview` (its mouse-rest preview), `NavGrid` (a plain grid, list or table body the keyboard walks — `DataTable`, `VirtualGrid` and `Carousel` take `keyboardNav`), `RemoteImage` (every network picture: no Referer, retries past the negative cache, a fallback instead of the broken-image mark — a bare `<img>` is linted out), `Avatar`, `CountBadge`, `ItemCount`, `FilterChipGroup`, `RelativeTime` ("3d ago" with the exact minute on hover, docs/DESIGN.md "Copy"), `ClampedText` (a long text clamped to a few lines; More / Less only when it is cut), `StatTiles` + `StatTile` / `StatBars` (number tiles, a breakdown as horizontal bars), `RangeTrack` + `trackPosition` + `RulerTicks` (a time track over a window of a media item: regions, slider ends the arrows step, a playhead, a background slot — the editor's cuts, the players' clip panel), `CropFrame` (a crop frame over a picture, on the picture and not its bars: drag, edges, corners, the arrows; the editor's crop, the players' clip crop), `reveal` (a card's hover chrome: hover or keyboard focus on a mouse, never on touch) |
|
||||
| page & panel structure | `PageShell` + `pageToolbarSlot` + `pageHeaderSlot` (`PageHeader`: a page fills the floating header) + `pageHeading`, `PageScroller`, `Headings` (`PageHeading`, `SectionLabel`), `SkipLink` + `landmarks`, `FilterFold` (B4), `scrollFold` (`useScrollFold`: a fixed header that folds once its content scrolls), `PanelGroup`, `PanelGroups`, `Tabs` + `tabPanel`, `BackToTop` + `backToTopState` (PageScroller keeps the page's end clear of it), `FilterRail` (the filter-rail shell a rail fills with its groups: title bar, customize, reset), `stickyTop` (a sticky header at the scroller's top turns its top fade off) |
|
||||
| state & messaging | `QueryState`, `Banner`, `InlineNotice`, `CrashBoundary` |
|
||||
|
||||
## The size contract (E10 S1c)
|
||||
|
||||
|
||||
@@ -52,10 +52,11 @@ const PLAIN: Record<string, [number, string]> = {
|
||||
"modules/settings/SettingsPanel.tsx": [1, "a colour swatch"],
|
||||
"modules/channels/SyncStatus.tsx": [1, "the status pill trigger"],
|
||||
"features/channel/TagManager.tsx": [1, "a suggestion in a picker"],
|
||||
"modules/files/UploadDialog.tsx": [1, "the drop zone"],
|
||||
"components/ui/DropZone.tsx": [1, "the drop zone (its text is its own prompt)"],
|
||||
"modules/files/VideoEditor.tsx": [2, "timeline glyphs"],
|
||||
"shell/Welcome.tsx": [1, "a screenshot (zoom)"],
|
||||
"modules/playlists/playlistColumns.tsx": [1, "the thumbnail"],
|
||||
"modules/channels/WatchedVideosDialog.tsx": [1, "the thumbnail (its title sits beside it)"],
|
||||
"modules/plex/plexDetailUi.tsx": [2, "a switch row and the poster"],
|
||||
"modules/plex/plexViews.tsx": [1, "the rating filter (a number)"],
|
||||
"components/ui/ActionMenu.tsx": [1, "a menu row"],
|
||||
|
||||
@@ -1,4 +1,26 @@
|
||||
[
|
||||
{
|
||||
"version": "0.111.0",
|
||||
"date": "2026-10-24",
|
||||
"new": [
|
||||
"Channels › Discover from your history: the channels your uploaded YouTube history shows you watching often (five videos of five minutes or more) but not following.",
|
||||
"Follow one, put it out of the list (with Undo), or open the videos you watched there and play them one by one or all in a row."
|
||||
],
|
||||
"fixed": [
|
||||
"A channel someone explored and then followed on YouTube is now part of the shared Library like any followed channel, and so are its videos; before, they could stay hidden and their watched marks never applied."
|
||||
]
|
||||
},
|
||||
{
|
||||
"version": "0.110.0",
|
||||
"date": "2026-10-23",
|
||||
"new": [
|
||||
"YouTube history: upload your Google Takeout export in Settings › Account. What you watched on YouTube is marked watched here (never a Short, never over a state you set), and your YouTube Watch later list joins yours.",
|
||||
"A video the catalog does not know yet is remembered and marked once it arrives. The import's outcome reaches your inbox, and admins see each run on the Scheduler page."
|
||||
],
|
||||
"fixed": [
|
||||
"Scheduler: a channel's full-history backfill asked for again while one is running now runs after it, instead of being silently dropped."
|
||||
]
|
||||
},
|
||||
{
|
||||
"version": "0.109.2",
|
||||
"date": "2026-10-23",
|
||||
|
||||
@@ -1,4 +1,26 @@
|
||||
[
|
||||
{
|
||||
"version": "0.111.0",
|
||||
"date": "2026-10-24",
|
||||
"new": [
|
||||
"Csatornák › Felfedezés az előzményekből: azok a csatornák, amelyeket a feltöltött YouTube-előzményeid szerint sokat nézel (öt legalább ötperces videó), de nem követsz.",
|
||||
"Követheted, kiveheted a listából (visszavonható), vagy megnyithatod az ott megnézett videóit, és egyenként vagy sorban lejátszhatod őket."
|
||||
],
|
||||
"fixed": [
|
||||
"Ha egy felfedezett csatornát valaki a YouTube-on kezd követni, az most a többi követett csatornához hasonlóan a közös Könyvtár része lesz a videóival együtt; eddig rejtve maradhattak, és a megnézett jelöléseik sem érvényesültek."
|
||||
]
|
||||
},
|
||||
{
|
||||
"version": "0.110.0",
|
||||
"date": "2026-10-23",
|
||||
"new": [
|
||||
"YouTube-előzmények: a Google Takeout exportod a Beállítások › Fiók oldalon tölthető fel. Amit a YouTube-on megnéztél, itt megnézettnek jelölődik (Short soha, és amit itt beállítottál, azt nem írja felül), a „Megnézem később” listád pedig az ittenihez adódik.",
|
||||
"Egy videót, amit a katalógus még nem ismer, megjegyez, és megjelöli, amint bekerül. Az import eredménye a postafiókodba érkezik, az adminok pedig minden futását látják az Ütemező oldalon."
|
||||
],
|
||||
"fixed": [
|
||||
"Ütemező: ha egy csatorna teljes előzményének letöltését futás közben újra kérik, az most utána lefut, nem vész el csendben."
|
||||
]
|
||||
},
|
||||
{
|
||||
"version": "0.109.2",
|
||||
"date": "2026-10-23",
|
||||
|
||||
@@ -305,6 +305,37 @@ describe("backend ids have their names in both languages", () => {
|
||||
expect([...RUN_STATUSES].sort()).toEqual([...runStatuses].sort());
|
||||
});
|
||||
|
||||
it("the Takeout import's notice is known by the server's job id", () => {
|
||||
// Read as text: the slice pulls in the request layer, which needs a browser.
|
||||
const feedTs = readFileSync(new URL("../lib/api/feed.ts", import.meta.url), "utf8");
|
||||
const ours = /^export const TAKEOUT_JOB = "(\w+)";$/m.exec(feedTs)?.[1];
|
||||
expect(ours).toBeTruthy();
|
||||
expect(/^TAKEOUT_IMPORT = "(\w+)"$/m.exec(schedulerPy)?.[1]).toBe(ours);
|
||||
});
|
||||
|
||||
it("the Takeout import's files and outcomes are the server's", () => {
|
||||
const feedTs = readFileSync(new URL("../lib/api/feed.ts", import.meta.url), "utf8");
|
||||
const ours = (name: string) =>
|
||||
[
|
||||
...(
|
||||
new RegExp(`^export const ${name} = \\[([^\\]]*)\\]`, "m").exec(feedTs)?.[1] ?? ""
|
||||
).matchAll(/"(\w+)"/g),
|
||||
].map((m) => m[1]);
|
||||
const models = backend("reader/models.py");
|
||||
const theirs = (cls: string) =>
|
||||
[
|
||||
...(
|
||||
new RegExp(`^class ${cls}\\(StrEnum\\):\\n([\\s\\S]*?)(?=\\n\\S)`, "m").exec(
|
||||
models,
|
||||
)?.[1] ?? ""
|
||||
).matchAll(/^\s+[A-Z_]+ = "(\w+)"/gm),
|
||||
].map((m) => m[1]);
|
||||
expect(theirs("TakeoutKind").length).toBeGreaterThan(0);
|
||||
expect(ours("TAKEOUT_KINDS")).toEqual(theirs("TakeoutKind"));
|
||||
expect(theirs("TakeoutOutcome").length).toBeGreaterThan(0);
|
||||
expect(ours("TAKEOUT_OUTCOMES")).toEqual(theirs("TakeoutOutcome"));
|
||||
});
|
||||
|
||||
for (const lang of ["en", "hu"]) {
|
||||
const keys = keysOf(lang);
|
||||
const missing = (wanted: string[]) => wanted.filter((k) => !keys.has(k));
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
"tabs": {
|
||||
"subscribed": "Subscriptions",
|
||||
"discovery": "Discover from playlists",
|
||||
"watched": "Discover from your history",
|
||||
"hidden": "Hidden",
|
||||
"blocked": "Blocked"
|
||||
},
|
||||
@@ -154,5 +155,38 @@
|
||||
"hint": "Their videos are kept out of your personal Feed, but still appear in the shared Library.",
|
||||
"empty": "No hidden channels. Hide a subscription (the eye icon on its row) to keep its uploads out of your personal Feed without unsubscribing.",
|
||||
"unhide": "Unhide"
|
||||
},
|
||||
"watched": {
|
||||
"intro": "Channels you often watched on YouTube but don't follow, from the watch history you uploaded. Only videos of five minutes or more count, so Shorts stay out. Follow a channel to get its new uploads in your Feed, or take it off the list.",
|
||||
"loading": "Finding channels…",
|
||||
"empty": "No suggestions — you follow every channel you watched five or more longer videos from.",
|
||||
"notImported": "Upload your YouTube watch history to see the channels you watch but don't follow.",
|
||||
"openTakeout": "Open YouTube history",
|
||||
"pending_one": "{{count}} more channel is still being looked at — the list may grow.",
|
||||
"pending_other": "{{count}} more channels are still being looked at — the list may grow.",
|
||||
"more_one": "Show {{count}} more",
|
||||
"more_other": "Show {{count}} more",
|
||||
"shown": "{{shown}} of {{total}}",
|
||||
"dismiss": "Not interested",
|
||||
"dismissHint": "Take this channel off the list. You can undo it right after.",
|
||||
"dismissed": "{{name}} is off the list",
|
||||
"cols": {
|
||||
"watched": "Watched",
|
||||
"watchedHint_one": "Show the {{count}} video of five minutes or more you watched from this channel",
|
||||
"watchedHint_other": "Show the {{count}} videos of five minutes or more you watched from this channel",
|
||||
"last": "Last watched"
|
||||
},
|
||||
"videos": {
|
||||
"title": "Videos you watched from {{name}}",
|
||||
"summary_one": "{{count}} video of five minutes or more · last watched {{when}}",
|
||||
"summary_other": "{{count}} videos of five minutes or more · last watched {{when}}",
|
||||
"loading": "Getting the videos ready…",
|
||||
"gone_one": "{{count}} video is no longer on YouTube",
|
||||
"gone_other": "{{count}} videos are no longer on YouTube",
|
||||
"empty": "None of these videos is on YouTube any more.",
|
||||
"colTitle": "Video",
|
||||
"colLength": "Length",
|
||||
"colWatched": "Watched"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,6 +87,7 @@
|
||||
"purgeDiscovery": "Couldn't clear the discovery data",
|
||||
"plexWatchLink": "Couldn't change the Plex watch-state sync",
|
||||
"plexWatchImport": "Couldn't import the watch state from Plex",
|
||||
"takeoutUpload": "Couldn't upload the Takeout file",
|
||||
"shareLinkDownload": "Couldn't change whether the link allows downloading",
|
||||
"shareLinkRevoke": "Couldn't revoke the link",
|
||||
"shareLinkPassword": "Couldn't change the link's password",
|
||||
@@ -114,7 +115,9 @@
|
||||
"apiToken": {
|
||||
"create": "Couldn't create the token",
|
||||
"revoke": "Couldn't revoke the token"
|
||||
}
|
||||
},
|
||||
"dismissWatchedChannel": "Couldn't take the channel off the list",
|
||||
"restoreWatchedChannel": "Couldn't put the channel back on the list"
|
||||
},
|
||||
"ok": "OK",
|
||||
"offline": {
|
||||
@@ -249,6 +252,13 @@
|
||||
"api_token_preset_grants": "The Hub token gets its folder by itself — leave the folders out",
|
||||
"api_token_no_grants": "Choose at least one folder the token may use",
|
||||
"api_token_name_required": "Give the token a name",
|
||||
"api_token_limit": "You already have {{limit}} tokens — revoke one first"
|
||||
"api_token_limit": "You already have {{limit}} tokens — revoke one first",
|
||||
"takeout_unreadable": "This isn't a Takeout file the import reads. Upload watch-history.json (JSON format) or Watch later-videos.csv.",
|
||||
"takeout_no_videos": "This file has no videos to import",
|
||||
"takeout_busy": "Another of your files is still being uploaded — wait for it to finish",
|
||||
"takeout_server_busy": "Other imports are being received right now — try again in a minute",
|
||||
"takeout_cooldown": "You have uploaded many files in the last hour — try again later",
|
||||
"takeout_too_many": "That would keep more than {{limit, number}} videos from your export — more than a real one holds",
|
||||
"takeout_list_too_long": "This list holds more than {{limit, number}} videos — more than a YouTube list can"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
"openScheduler": "Open Scheduler",
|
||||
"openFile": "Open the file",
|
||||
"openTokens": "Open API tokens",
|
||||
"openTakeout": "Open YouTube history",
|
||||
"revertFailed": "Couldn't undo that — please try again",
|
||||
"downloadDone": {
|
||||
"title": "Download ready"
|
||||
|
||||
@@ -67,7 +67,7 @@
|
||||
"enrich": "Fetches video details (duration, views, live/Shorts flags) and re-checks live videos until they end. If it stops, videos stay without a duration/stats and ended livestreams stay stuck as “live”.",
|
||||
"backfill": "Pulls in recent uploads, then the full back-catalogue for opted-in channels, within the shared quota. If it stops, older videos and full history don't fill in.",
|
||||
"autotag": "Detects each channel's language/topics and applies automatic tags. If it stops, new channels get no auto tags, so language/topic filters miss them.",
|
||||
"shorts": "Probes youtube.com/shorts to mark which videos are Shorts. If it stops, Shorts aren't separated from normal videos in the Feed.",
|
||||
"shorts": "Probes youtube.com/shorts to mark which videos are Shorts, then marks watched the videos a Google Takeout import was waiting on. If it stops, Shorts aren't separated from normal videos in the Feed, and those videos keep waiting.",
|
||||
"subscriptions": "Re-imports every user's YouTube subscriptions. If it stops, channels subscribed to or dropped on YouTube aren't reflected here.",
|
||||
"playlist_sync": "Mirrors each user's YouTube playlists into the app. If it stops, changes to your YouTube playlists don't show up locally.",
|
||||
"maintenance": "Finds videos that can no longer be played (deleted, made private, abandoned premieres), hides them from the Feed and removes them after a grace period; re-checks the least-recently-checked videos. If it stops, dead videos linger in the catalogue.",
|
||||
@@ -80,7 +80,8 @@
|
||||
"log_gc": "Deletes stored log events older than the retention window (Configuration → Diagnostics). Runs hourly so nothing outlives the promised limit by more than an hour.",
|
||||
"tmdb_enrich": "Enriches the mirrored Plex catalogue from TMDB (keywords, crew, budget/revenue, TMDb score, recommendations, artwork), matched by each title's canonical id and served from the app's own database. Optional — a no-op until a TMDB API key is set. If it stops, new Plex titles just miss their TMDB extras until it runs again.",
|
||||
"people_enrich": "Makes the people in the Plex library local: fetches each person's biography from TMDB and caches their portrait, resized through Plex, the most-credited people first. A person page and a cast strip then open from the app's own data. The TMDB half does nothing without a TMDB API key, the portrait half nothing without Plex. If it stops, a person page fetches its own bio when opened, a moment later.",
|
||||
"channel_backfill": "Turning on full history for a channel fetches its whole back-catalogue at once, as its own run. The Backfill job finishes what this one could not (a very large channel, low quota, or two of these already running)."
|
||||
"channel_backfill": "Turning on full history for a channel fetches its whole back-catalogue at once, as its own run. The Backfill job finishes what this one could not (a very large channel, low quota, or two of these already running).",
|
||||
"takeout_import": "A user's Google Takeout import: what they watched on YouTube becomes watched here (not Shorts, never over a state they set here), and their YouTube Watch later list joins theirs. A video the catalogue does not know yet waits, and is marked once it arrives."
|
||||
},
|
||||
"jobs": {
|
||||
"rss_poll": "RSS poll (new uploads)",
|
||||
@@ -100,7 +101,8 @@
|
||||
"audit_gc": "Activity and quota-history cleanup",
|
||||
"log_gc": "Log cleanup",
|
||||
"people_enrich": "People details and portraits (Plex)",
|
||||
"channel_backfill": "A channel's full history"
|
||||
"channel_backfill": "A channel's full history",
|
||||
"takeout_import": "Google Takeout import"
|
||||
},
|
||||
"queue": {
|
||||
"recentPending": "Channels to sync",
|
||||
@@ -178,7 +180,16 @@
|
||||
"deep": "deep",
|
||||
"bios": "biographies",
|
||||
"portraits": "portraits",
|
||||
"cached": "cached"
|
||||
"cached": "cached",
|
||||
"kept": "kept as set here",
|
||||
"waiting": "waiting for their video",
|
||||
"watch_later_added": "added to Watch later",
|
||||
"watch_later_present": "already in Watch later",
|
||||
"watch_later_missing": "no longer on YouTube",
|
||||
"watch_later_waiting": "Watch later waiting",
|
||||
"takeout_watched": "marked watched from Takeout",
|
||||
"takeout_watch_later_added": "put into Watch later from Takeout",
|
||||
"takeout_failed": "Takeout users left for the next run after an error"
|
||||
},
|
||||
"skip": {
|
||||
"disabled": "disabled",
|
||||
@@ -191,11 +202,13 @@
|
||||
"busy": "another sync is running",
|
||||
"channelbusy": "this channel is already being backfilled",
|
||||
"atcapacity": "two full-history backfills are already running — the scheduler finishes this one",
|
||||
"nothingtodo": "nothing left to fetch"
|
||||
"nothingtodo": "nothing left to fetch",
|
||||
"nouser": "the account is gone"
|
||||
},
|
||||
"failure": {
|
||||
"rss_all_failed_one": "The RSS feed did not answer for the one channel, {{channel}}: {{cause}}. Read through the API instead: {{read}}.",
|
||||
"rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}."
|
||||
"rss_all_failed_other": "The RSS feed answered for none of the {{total}} channels — the first: {{channel}}, {{cause}}. Read through the API instead: {{read}}.",
|
||||
"unexpected": "The run ran into an unexpected error."
|
||||
},
|
||||
"rssCause": {
|
||||
"http": "HTTP {{status}}",
|
||||
|
||||
@@ -19,6 +19,42 @@
|
||||
"never": "Not imported yet",
|
||||
"importing": "Importing your Plex watch history…"
|
||||
},
|
||||
"takeout": {
|
||||
"title": "YouTube history",
|
||||
"intro": "Upload “watch-history.json” from your Google Takeout export (and “Watch later-videos.csv”, if you have it). Videos known here show as watched; the rest are remembered and marked watched once they arrive here. Shorts are skipped, and nothing you set here is overwritten.",
|
||||
"drop": "Drop the files here, or choose them",
|
||||
"dropActive": "Drop to upload",
|
||||
"uploading": "Uploading {{name}}…",
|
||||
"read_one": "{{name}}: {{n}} video read",
|
||||
"read_other": "{{name}}: {{n}} videos read",
|
||||
"running": "Processing…",
|
||||
"last": "Last import: {{when}}",
|
||||
"failed": "The last import ({{when}}) failed. Upload the file again to retry.",
|
||||
"interrupted": "The last import ({{when}}) was cut off by a server restart. Upload the file again to finish it.",
|
||||
"newImport": "New import",
|
||||
"loadFailed": "Couldn't load the last import",
|
||||
"howTo": "How to export from Takeout?",
|
||||
"howToBody": "At takeout.google.com select only “YouTube and YouTube Music”. In its options keep only “history” and “playlists”, and choose JSON for the history. Untick “videos”, or every video you ever uploaded comes with it.",
|
||||
"count": {
|
||||
"watched_one": "{{n}} watched",
|
||||
"watched_other": "{{n}} watched",
|
||||
"waiting_one": "{{n}} remembered",
|
||||
"waiting_other": "{{n}} remembered",
|
||||
"short_one": "{{n}} Short skipped",
|
||||
"short_other": "{{n}} Shorts skipped",
|
||||
"kept_one": "{{n}} left as you set it here",
|
||||
"kept_other": "{{n}} left as you set them here",
|
||||
"added_one": "{{n}} added to Watch later",
|
||||
"added_other": "{{n}} added to Watch later",
|
||||
"present_one": "{{n}} already in Watch later",
|
||||
"present_other": "{{n}} already in Watch later",
|
||||
"missing_one": "{{n}} no longer on YouTube",
|
||||
"missing_other": "{{n}} no longer on YouTube",
|
||||
"laterWaiting_one": "{{n}} waiting for Watch later",
|
||||
"laterWaiting_other": "{{n}} waiting for Watch later"
|
||||
},
|
||||
"tooLarge": "{{name}} is larger than {{limit}} — the import takes files up to that size"
|
||||
},
|
||||
"save": {
|
||||
"saving": "Saving…",
|
||||
"saved": "Settings saved",
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
"tabs": {
|
||||
"subscribed": "Feliratkozások",
|
||||
"discovery": "Felfedezés a lejátszási listákból",
|
||||
"watched": "Felfedezés az előzményekből",
|
||||
"hidden": "Elrejtett",
|
||||
"blocked": "Letiltott"
|
||||
},
|
||||
@@ -154,5 +155,38 @@
|
||||
"hint": "A videóik kimaradnak a személyes Feededből, de a megosztott Könyvtárban továbbra is megjelennek.",
|
||||
"empty": "Nincs elrejtett csatorna. Egy feliratkozást a sorában (a szem ikonnal) rejthetsz el, hogy kimaradjon a személyes Feededből leiratkozás nélkül.",
|
||||
"unhide": "Felfedés"
|
||||
},
|
||||
"watched": {
|
||||
"intro": "Csatornák, amelyeket YouTube-on gyakran néztél, de nem követsz — a feltöltött nézési előzményeid alapján. Csak az öt percnél hosszabb videók számítanak, így a Shorts kimarad. Kövesd a csatornát, és az új videói megjelennek a hírfolyamodban, vagy vedd le a listáról.",
|
||||
"loading": "Csatornák keresése…",
|
||||
"empty": "Nincs javaslat — minden csatornát követsz, amelyről legalább öt hosszabb videót néztél.",
|
||||
"notImported": "Töltsd fel a YouTube nézési előzményeidet, és itt látod a csatornákat, amelyeket nézel, de nem követsz.",
|
||||
"openTakeout": "YouTube-előzmények megnyitása",
|
||||
"pending_one": "Még {{count}} csatornát vizsgálunk — a lista még bővülhet.",
|
||||
"pending_other": "Még {{count}} csatornát vizsgálunk — a lista még bővülhet.",
|
||||
"more_one": "További {{count}}",
|
||||
"more_other": "További {{count}}",
|
||||
"shown": "{{shown}} / {{total}}",
|
||||
"dismiss": "Nem érdekel",
|
||||
"dismissHint": "A csatorna lekerül a listáról. Utána rögtön visszavonhatod.",
|
||||
"dismissed": "{{name}} lekerült a listáról",
|
||||
"cols": {
|
||||
"watched": "Nézett",
|
||||
"watchedHint_one": "A csatornáról nézett {{count}}, öt percnél hosszabb videó megnyitása",
|
||||
"watchedHint_other": "A csatornáról nézett {{count}}, öt percnél hosszabb videó megnyitása",
|
||||
"last": "Utoljára nézve"
|
||||
},
|
||||
"videos": {
|
||||
"title": "{{name}}: a videók, amelyeket láttál",
|
||||
"summary_one": "{{count}} öt percnél hosszabb videó · utoljára nézve: {{when}}",
|
||||
"summary_other": "{{count}} öt percnél hosszabb videó · utoljára nézve: {{when}}",
|
||||
"loading": "A videók előkészítése…",
|
||||
"gone_one": "{{count}} videó már nincs fent a YouTube-on",
|
||||
"gone_other": "{{count}} videó már nincs fent a YouTube-on",
|
||||
"empty": "Ezek közül egyik videó sincs már fent a YouTube-on.",
|
||||
"colTitle": "Videó",
|
||||
"colLength": "Hossz",
|
||||
"colWatched": "Nézve"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -87,6 +87,7 @@
|
||||
"purgeDiscovery": "Nem sikerült törölni a felfedezési adatokat",
|
||||
"plexWatchLink": "Nem sikerült módosítani a Plex megtekintésiállapot-szinkront",
|
||||
"plexWatchImport": "Nem sikerült importálni a megtekintési állapotot a Plexből",
|
||||
"takeoutUpload": "Nem sikerült feltölteni a Takeout-fájlt",
|
||||
"shareLinkDownload": "Nem sikerült módosítani, hogy a link engedi-e a letöltést",
|
||||
"shareLinkRevoke": "Nem sikerült visszavonni a linket",
|
||||
"shareLinkPassword": "Nem sikerült módosítani a link jelszavát",
|
||||
@@ -114,7 +115,9 @@
|
||||
"apiToken": {
|
||||
"create": "Nem sikerült létrehozni a tokent",
|
||||
"revoke": "Nem sikerült visszavonni a tokent"
|
||||
}
|
||||
},
|
||||
"dismissWatchedChannel": "Nem sikerült levenni a csatornát a listáról",
|
||||
"restoreWatchedChannel": "Nem sikerült visszatenni a csatornát a listára"
|
||||
},
|
||||
"ok": "OK",
|
||||
"offline": {
|
||||
@@ -249,6 +252,13 @@
|
||||
"api_token_preset_grants": "A Hub-token magától kapja a mappáját — ne adj meg mappát",
|
||||
"api_token_no_grants": "Válassz legalább egy mappát, amelyet a token használhat",
|
||||
"api_token_name_required": "Adj nevet a tokennek",
|
||||
"api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet"
|
||||
"api_token_limit": "Már {{limit}} tokened van — előbb vonj vissza egyet",
|
||||
"takeout_unreadable": "Ez nem olyan Takeout-fájl, amit az import olvas. A watch-history.json (JSON formátum) vagy a Watch later-videos.csv fájlt töltsd fel.",
|
||||
"takeout_no_videos": "Ebben a fájlban nincs importálható videó",
|
||||
"takeout_busy": "Egy másik fájlod feltöltése még folyik — várd meg, amíg befejeződik",
|
||||
"takeout_server_busy": "Épp más importok érkeznek — próbáld újra egy perc múlva",
|
||||
"takeout_cooldown": "Az elmúlt órában sok fájlt töltöttél fel — próbáld újra később",
|
||||
"takeout_too_many": "Így több mint {{limit, number}} videót őriznénk meg az exportodból — többet, mint amennyi egy valódiban van",
|
||||
"takeout_list_too_long": "Ez a lista több mint {{limit, number}} videót tartalmaz — többet, mint amennyi egy YouTube-listán lehet"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
"openScheduler": "Ütemező megnyitása",
|
||||
"openFile": "Fájl megnyitása",
|
||||
"openTokens": "API-tokenek megnyitása",
|
||||
"openTakeout": "YouTube-előzmények megnyitása",
|
||||
"revertFailed": "Nem sikerült visszavonni — próbáld újra",
|
||||
"downloadDone": {
|
||||
"title": "Letöltés kész"
|
||||
|
||||
@@ -67,7 +67,7 @@
|
||||
"enrich": "Lekéri a videók adatait (hossz, megtekintés, élő/Shorts jelzők), és frissíti az élő videókat, amíg véget nem érnek. Ha leáll, a videók hossz/statisztika nélkül maradnak, a véget ért adások pedig „élő” állapotban ragadnak.",
|
||||
"backfill": "Behúzza a friss feltöltéseket, majd a kijelölt csatornák teljes előzményét, a közös kvótán belül. Ha leáll, a régebbi videók és a teljes előzmény nem töltődik fel.",
|
||||
"autotag": "Felismeri a csatornák nyelvét/témáit, és automatikus címkéket ad. Ha leáll, az új csatornák címke nélkül maradnak, így a nyelv/téma szűrők kihagyják őket.",
|
||||
"shorts": "A youtube.com/shorts próbával jelöli, mely videók Shorts-ok. Ha leáll, a Shorts-ok nem különülnek el a normál videóktól a Feedben.",
|
||||
"shorts": "A youtube.com/shorts próbával jelöli, mely videók Shorts-ok, utána megnézettre jelöli azokat a videókat, amelyekre egy Google Takeout import várt. Ha leáll, a Shorts-ok nem különülnek el a normál videóktól a Feedben, és ezek a videók tovább várnak.",
|
||||
"subscriptions": "Újraimportálja minden felhasználó YouTube-feliratkozásait. Ha leáll, a YouTube-on felvett vagy törölt feliratkozások nem tükröződnek itt.",
|
||||
"playlist_sync": "Tükrözi a felhasználók YouTube lejátszási listáit az appba. Ha leáll, a YouTube-listák változásai nem jelennek meg lokálisan.",
|
||||
"maintenance": "Megkeresi a már sehol le nem játszható videókat (törölt, priváttá tett, elhagyott premier), elrejti őket a Feedből, és türelmi idő után eltávolítja; újraellenőrzi a legrégebben ellenőrzött videókat. Ha leáll, a halott videók a katalógusban maradnak.",
|
||||
@@ -80,7 +80,8 @@
|
||||
"log_gc": "Törli a megőrzési időnél (Konfiguráció → Diagnosztika) régebbi tárolt naplóeseményeket. Óránként fut, így semmi sem marad meg egy óránál tovább az ígért határidőn túl.",
|
||||
"tmdb_enrich": "A tükrözött Plex-katalógust dúsítja a TMDB-ből (kulcsszavak, stáb, költségvetés/bevétel, TMDb-pontszám, ajánlások, artwork), a címek kanonikus azonosítója alapján párosítva, és az app saját adatbázisából szolgálja ki. Opcionális — TMDB API-kulcs nélkül nem csinál semmit. Ha leáll, az új Plex-címek csak a következő futásig maradnak TMDB-extrák nélkül.",
|
||||
"people_enrich": "Helyivé teszi a Plex-könyvtár szereplőit: a TMDB-ből letölti az életrajzukat, a portréjukat pedig a Plexszel kicsinyítve elmenti, a legtöbbet szereplőkkel kezdve. A személyoldal és a szereplősáv így az app saját adataiból nyílik meg. A TMDB-rész TMDB API-kulcs nélkül nem csinál semmit, a portrérész Plex nélkül. Ha leáll, a személyoldal megnyitáskor maga kéri le az életrajzot, egy pillanattal később.",
|
||||
"channel_backfill": "Ha egy csatornánál bekapcsolod a teljes előzményt, a teljes korábbi kínálata egyszerre, saját futásként töltődik le. Amit ez nem tud befejezni (egy nagyon nagy csatorna, kevés kvóta, vagy ha már két ilyen fut), azt a „Videók behúzása” feladat fejezi be."
|
||||
"channel_backfill": "Ha egy csatornánál bekapcsolod a teljes előzményt, a teljes korábbi kínálata egyszerre, saját futásként töltődik le. Amit ez nem tud befejezni (egy nagyon nagy csatorna, kevés kvóta, vagy ha már két ilyen fut), azt a „Videók behúzása” feladat fejezi be.",
|
||||
"takeout_import": "Egy felhasználó Google Takeout importja: amit a YouTube-on megnézett, itt is megnézett lesz (a Shorts nem, és amit itt beállított, azt nem írja felül), a YouTube Watch later listája pedig bekerül a „Megnézem később” listájába. A katalógusban még nem szereplő videó vár, és amint megérkezik, megnézett lesz."
|
||||
},
|
||||
"jobs": {
|
||||
"rss_poll": "RSS-lekérdezés (új feltöltések)",
|
||||
@@ -100,7 +101,8 @@
|
||||
"audit_gc": "Napló- és kvótaelőzmény-takarítás",
|
||||
"log_gc": "Naplótakarítás",
|
||||
"people_enrich": "Személyadatok és portrék (Plex)",
|
||||
"channel_backfill": "Egy csatorna teljes előzménye"
|
||||
"channel_backfill": "Egy csatorna teljes előzménye",
|
||||
"takeout_import": "Google Takeout import"
|
||||
},
|
||||
"queue": {
|
||||
"recentPending": "Szinkronra váró csatorna",
|
||||
@@ -178,7 +180,16 @@
|
||||
"deep": "mély",
|
||||
"bios": "életrajzok",
|
||||
"portraits": "portrék",
|
||||
"cached": "elmentve"
|
||||
"cached": "elmentve",
|
||||
"kept": "itteni állapot megtartva",
|
||||
"waiting": "a videójára vár",
|
||||
"watch_later_added": "a „Megnézem később” listába téve",
|
||||
"watch_later_present": "már a „Megnézem később” listán",
|
||||
"watch_later_missing": "már nincs a YouTube-on",
|
||||
"watch_later_waiting": "a „Megnézem később” listára vár",
|
||||
"takeout_watched": "Takeoutból megnézettre jelölve",
|
||||
"takeout_watch_later_added": "Takeoutból a „Megnézem később” listába téve",
|
||||
"takeout_failed": "hiba miatt a következő futásra maradt Takeout-felhasználók"
|
||||
},
|
||||
"skip": {
|
||||
"disabled": "kikapcsolva",
|
||||
@@ -191,11 +202,13 @@
|
||||
"busy": "egy másik szinkron fut",
|
||||
"channelbusy": "ennek a csatornának a betöltése már fut",
|
||||
"atcapacity": "már két teljes előzmény-betöltés fut — ezt az ütemező fejezi be",
|
||||
"nothingtodo": "nincs több letöltendő"
|
||||
"nothingtodo": "nincs több letöltendő",
|
||||
"nouser": "a fiók már nem létezik"
|
||||
},
|
||||
"failure": {
|
||||
"rss_all_failed_one": "Az RSS-hírcsatorna az egyetlen csatornánál sem válaszolt ({{channel}}): {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
|
||||
"rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}."
|
||||
"rss_all_failed_other": "Az RSS-hírcsatorna egyik csatornánál sem válaszolt ({{total}}) — az első: {{channel}}, {{cause}}. Az API-n át beolvasva helyette: {{read}}.",
|
||||
"unexpected": "A futás váratlan hibába ütközött."
|
||||
},
|
||||
"rssCause": {
|
||||
"http": "HTTP {{status}}",
|
||||
|
||||
@@ -19,6 +19,42 @@
|
||||
"never": "Még nincs importálva",
|
||||
"importing": "Plex-előzmény importálása…"
|
||||
},
|
||||
"takeout": {
|
||||
"title": "YouTube-előzmények",
|
||||
"intro": "Töltsd fel a Google Takeout exportodból a „watch-history.json” fájlt (és ha van, a „Watch later-videos.csv” fájlt). Az itt ismert videók megnézettként jelennek meg; a többit megjegyezzük, és megnézettek lesznek, amint bekerülnek ide. A Shortsot kihagyjuk, és amit itt beállítottál, azt nem írjuk felül.",
|
||||
"drop": "Húzd ide a fájlokat, vagy válaszd ki őket",
|
||||
"dropActive": "Engedd el a feltöltéshez",
|
||||
"uploading": "{{name}} feltöltése…",
|
||||
"read_one": "{{name}}: {{n}} videó beolvasva",
|
||||
"read_other": "{{name}}: {{n}} videó beolvasva",
|
||||
"running": "Feldolgozás…",
|
||||
"last": "Utolsó import: {{when}}",
|
||||
"failed": "Az utolsó import ({{when}}) nem sikerült. Töltsd fel újra a fájlt.",
|
||||
"interrupted": "Az utolsó importot ({{when}}) megszakította a szerver újraindulása. Töltsd fel újra a fájlt a befejezéshez.",
|
||||
"newImport": "Új import",
|
||||
"loadFailed": "Nem sikerült betölteni az utolsó importot",
|
||||
"howTo": "Hogyan exportálj a Takeoutból?",
|
||||
"howToBody": "A takeout.google.com oldalon csak a „YouTube és YouTube Music” kell. A beállításai közül csak az „előzmények” és a „lejátszási listák” maradjon, az előzményekhez JSON formátummal. A „videók” pipát vedd ki, különben az összes feltöltött videód is benne lesz.",
|
||||
"count": {
|
||||
"watched_one": "{{n}} megnézett",
|
||||
"watched_other": "{{n}} megnézett",
|
||||
"waiting_one": "{{n}} megjegyezve",
|
||||
"waiting_other": "{{n}} megjegyezve",
|
||||
"short_one": "{{n}} Short kihagyva",
|
||||
"short_other": "{{n}} Short kihagyva",
|
||||
"kept_one": "{{n}} úgy maradt, ahogy itt beállítottad",
|
||||
"kept_other": "{{n}} úgy maradt, ahogy itt beállítottad",
|
||||
"added_one": "{{n}} a „Megnézem később” listába téve",
|
||||
"added_other": "{{n}} a „Megnézem később” listába téve",
|
||||
"present_one": "{{n}} már a „Megnézem később” listán",
|
||||
"present_other": "{{n}} már a „Megnézem később” listán",
|
||||
"missing_one": "{{n}} már nincs fent a YouTube-on",
|
||||
"missing_other": "{{n}} már nincs fent a YouTube-on",
|
||||
"laterWaiting_one": "{{n}} vár a „Megnézem később” listára",
|
||||
"laterWaiting_other": "{{n}} vár a „Megnézem később” listára"
|
||||
},
|
||||
"tooLarge": "{{name}}: nagyobb, mint {{limit}} — az import legfeljebb ekkora fájlt fogad"
|
||||
},
|
||||
"save": {
|
||||
"saving": "Mentés…",
|
||||
"saved": "Beállítások elmentve",
|
||||
|
||||
@@ -20,6 +20,7 @@ import { setupApi } from "./api/setup";
|
||||
export type * from "./api/setup";
|
||||
import { feedApi } from "./api/feed";
|
||||
export type * from "./api/feed";
|
||||
export { TAKEOUT_JOB } from "./api/feed";
|
||||
import { discoverApi } from "./api/discover";
|
||||
export type * from "./api/discover";
|
||||
export { DISCOVER_BUDGETS, HERO_TIERS, discoverCardKey } from "./api/discover";
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
// Feed, video watch-state/progress, and live YouTube search. `filterParams`/`feedQuery` translate
|
||||
// the shared FeedFilters into the query string the feed + facet endpoints both consume; they stay
|
||||
// module-local (no consumer names them). Only touches core's req/beacon.
|
||||
import { req, beacon } from "./core";
|
||||
import { req, beacon, uploadReq } from "./core";
|
||||
import type { RunStatus } from "./admin";
|
||||
import { TAG_CATEGORIES, type TagModes } from "./tagModes";
|
||||
|
||||
// Per-user watch status stored server-side (VALID_STATES in routes/feed.py). "in_progress" and
|
||||
@@ -148,6 +149,79 @@ function feedQuery(f: FeedFilters, cursor: string | null, limit: number): string
|
||||
return p.toString();
|
||||
}
|
||||
|
||||
// The import's job (`scheduler.TAKEOUT_IMPORT`, held to it by backendIds.contract.test): its notice
|
||||
// reaches a user who may not see the Scheduler, so it opens their own import instead.
|
||||
export const TAKEOUT_JOB = "takeout_import";
|
||||
|
||||
// A Google Takeout file, uploaded: which one it was, the videos read from it, and its entries left
|
||||
// out (an ad, a view of something that is not a video, a row that could not be read).
|
||||
export interface TakeoutUpload {
|
||||
ok: true;
|
||||
kind: TakeoutKind;
|
||||
videos: number;
|
||||
entries: number;
|
||||
ads: number;
|
||||
other: number;
|
||||
malformed: number;
|
||||
}
|
||||
// The two files (`reader.models.TakeoutKind`) and a kept video's outcome (`TakeoutOutcome`): watched
|
||||
// / kept (the user's own state here stood) / short for the history; added / present / missing for
|
||||
// Watch later. backendIds.contract.test holds both lists to the backend's enums.
|
||||
export const TAKEOUT_KINDS = ["watched", "watch_later"] as const;
|
||||
export const TAKEOUT_OUTCOMES = [
|
||||
"watched",
|
||||
"kept",
|
||||
"short",
|
||||
"added",
|
||||
"present",
|
||||
"missing",
|
||||
] as const;
|
||||
export type TakeoutKind = (typeof TAKEOUT_KINDS)[number];
|
||||
// Each outcome's count, and "waiting" for the videos the import could not settle yet.
|
||||
export type TakeoutCounts = Partial<Record<(typeof TAKEOUT_OUTCOMES)[number] | "waiting", number>>;
|
||||
export interface TakeoutStatus {
|
||||
/** An import is queued and its run not started: `run` is still the one before it. */
|
||||
queued: boolean;
|
||||
/** The largest file the upload takes. */
|
||||
max_bytes: number;
|
||||
run: {
|
||||
status: RunStatus;
|
||||
started_at: string | null;
|
||||
finished_at: string | null;
|
||||
result: Record<string, unknown> | null;
|
||||
failure: { failure: string; [key: string]: unknown } | null;
|
||||
} | null;
|
||||
watched?: TakeoutCounts;
|
||||
watch_later?: TakeoutCounts;
|
||||
}
|
||||
|
||||
// A channel the user's Takeout history shows them watching often but not following
|
||||
// (`GET /api/takeout/channels`): `watched` counts its videos of five minutes or more they watched.
|
||||
export interface WatchedChannel {
|
||||
id: string;
|
||||
title: string | null;
|
||||
handle: string | null;
|
||||
thumbnail_url: string | null;
|
||||
subscriber_count: number | null;
|
||||
video_count: number | null;
|
||||
description: string | null;
|
||||
watched: number;
|
||||
last_watched_at: string | null;
|
||||
}
|
||||
export interface WatchedChannels {
|
||||
/** Whether a watch history was uploaded at all ("nothing to suggest" vs "nothing to go on"). */
|
||||
imported: boolean;
|
||||
/** How many more channels may join once the import has asked YouTube for their videos' lengths. */
|
||||
pending: number;
|
||||
channels: WatchedChannel[];
|
||||
}
|
||||
/** A channel's counted videos, ready to play, last watched first; `gone`: those YouTube lost. */
|
||||
export interface WatchedChannelVideos {
|
||||
ok: true;
|
||||
items: (Video & { takeout_watched_at: string | null })[];
|
||||
gone: number;
|
||||
}
|
||||
|
||||
export const feedApi = {
|
||||
feed: (f: FeedFilters, cursor: string | null, limit: number): Promise<FeedResponse> =>
|
||||
req(`/api/feed?${feedQuery(f, cursor, limit)}`),
|
||||
@@ -210,4 +284,14 @@ export const feedApi = {
|
||||
duration_seconds: Math.floor(durationSeconds),
|
||||
}),
|
||||
videoDetail: (id: string): Promise<VideoDetail> => req(`/api/videos/${id}`),
|
||||
// One file of a Google Takeout export, its bytes as they are; the server tells the two apart.
|
||||
takeoutUpload: (file: File): Promise<TakeoutUpload> => uploadReq("/api/takeout", file),
|
||||
takeoutStatus: (): Promise<TakeoutStatus> => req("/api/takeout"),
|
||||
watchedChannels: (): Promise<WatchedChannels> => req("/api/takeout/channels"),
|
||||
watchedChannelVideos: (id: string): Promise<WatchedChannelVideos> =>
|
||||
req(`/api/takeout/channels/${encodeURIComponent(id)}/videos`, { method: "POST" }),
|
||||
dismissWatchedChannel: (id: string) =>
|
||||
req(`/api/takeout/channels/${encodeURIComponent(id)}/dismissal`, { method: "PUT" }),
|
||||
restoreWatchedChannel: (id: string) =>
|
||||
req(`/api/takeout/channels/${encodeURIComponent(id)}/dismissal`, { method: "DELETE" }),
|
||||
};
|
||||
|
||||
@@ -37,6 +37,10 @@ export const qk = {
|
||||
channels: () => ["channels"] as const,
|
||||
channel: (id: string) => ["channel", id] as const,
|
||||
discoveredChannels: () => ["discovered-channels"] as const,
|
||||
watchedChannels: () => ["watched-channels"] as const,
|
||||
// A root of its own, not under the list's: asking for them explores the channel (a write), so the
|
||||
// list's invalidations must not refetch them under an open dialog.
|
||||
watchedChannelVideos: (id: string) => ["watched-channel-videos", id] as const,
|
||||
blockedChannels: () => ["blocked-channels"] as const,
|
||||
|
||||
// --- playlists ---
|
||||
@@ -129,6 +133,7 @@ export const qk = {
|
||||
comfortPins: (lang?: string) => ["comfort-pins", ...(lang ? [lang] : [])] as const,
|
||||
plex: () => ["plex"] as const,
|
||||
plexWatchLink: () => ["plex-watch-link"] as const,
|
||||
takeout: () => ["takeout"] as const,
|
||||
plexLibrary: (...rest: unknown[]) => ["plex-library", ...rest],
|
||||
// Discover surface: the rail payload keyed by scope+lang; `plexDiscover()` (no args) matches it and
|
||||
// the Surprise draw below for a prefix invalidation after a watched-toggle.
|
||||
|
||||
@@ -20,6 +20,7 @@ describe("a subscription change", () => {
|
||||
qk.feed(),
|
||||
qk.feedSummary(),
|
||||
qk.discoveredChannels(),
|
||||
qk.watchedChannels(),
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -3,8 +3,8 @@ import type { QueryClient } from "@tanstack/react-query";
|
||||
import { qk } from "./queryKeys";
|
||||
|
||||
/** Everything a subscribe or an unsubscribe changes: the channel itself, the manager's list and its
|
||||
* counts, the feed and its facets (the channel's videos come or go), and discovery (a subscribed
|
||||
* channel leaves it). ONE set for every site that makes the change: each used to keep
|
||||
* counts, the feed and its facets (the channel's videos come or go), and the two discovery lists (a
|
||||
* subscribed channel leaves them). ONE set for every site that makes the change: each used to keep
|
||||
* its own list, and each list missed something. */
|
||||
export function invalidateSubscription(qc: QueryClient, channelId: string): void {
|
||||
for (const queryKey of [
|
||||
@@ -14,6 +14,7 @@ export function invalidateSubscription(qc: QueryClient, channelId: string): void
|
||||
qk.feed(),
|
||||
qk.feedSummary(),
|
||||
qk.discoveredChannels(),
|
||||
qk.watchedChannels(),
|
||||
])
|
||||
void qc.invalidateQueries({ queryKey });
|
||||
}
|
||||
|
||||
@@ -57,6 +57,7 @@ import { menuGroups } from "../../components/ui/menuGroups";
|
||||
import { choiceSubmenu, sortSubmenu } from "../../components/ui/pageMenuItems";
|
||||
import { optionsFromColumns } from "../../lib/sortOptions";
|
||||
import ChannelDiscovery from "./ChannelDiscovery";
|
||||
import WatchedChannels from "./WatchedChannels";
|
||||
import TagManager from "../../features/channel/TagManager";
|
||||
import {
|
||||
useChannelLayout,
|
||||
@@ -183,6 +184,8 @@ export default function Channels() {
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: qk.blockedChannels() });
|
||||
qc.invalidateQueries({ queryKey: qk.feed() });
|
||||
// A blocked channel is off the watched-channels list; unblocked, it may be back on it.
|
||||
qc.invalidateQueries({ queryKey: qk.watchedChannels() });
|
||||
},
|
||||
});
|
||||
|
||||
@@ -530,7 +533,7 @@ export default function Channels() {
|
||||
<ActionMenu ariaLabel={t("channels.actionsLabel")} items={catalogActions} />
|
||||
);
|
||||
|
||||
// Table / cards density — shared by the subscribed + discovery tables (blocked is a chip list, no
|
||||
// Table / cards density — shared by the subscribed + both discovery tables (blocked is a chip list, no
|
||||
// layout to switch). Rides the same generic ViewSwitcher the feed uses.
|
||||
const layoutOptions: ViewOption<ChannelLayout>[] = CHANNEL_LAYOUTS.map((id) => ({
|
||||
id,
|
||||
@@ -546,7 +549,7 @@ export default function Channels() {
|
||||
labelClassName="hidden lg:inline"
|
||||
/>
|
||||
);
|
||||
const hasLayout = view === "subscribed" || view === "discovery";
|
||||
const hasLayout = view === "subscribed" || view === "discovery" || view === "watched";
|
||||
// In cards there are no column headers to sort by, so a dropdown drives the SAME shared sort state
|
||||
// the table header uses (table ↔ cards stay in sync). Table mode keeps the header clicks.
|
||||
const sortControl = (
|
||||
@@ -612,7 +615,7 @@ export default function Channels() {
|
||||
|
||||
const viewTabs: TabDef[] = useMemo(
|
||||
() =>
|
||||
(["subscribed", "discovery", "hidden", "blocked"] as ChannelsView[]).map((v) => ({
|
||||
(["subscribed", "discovery", "watched", "hidden", "blocked"] as ChannelsView[]).map((v) => ({
|
||||
id: v,
|
||||
label: t(`channels.tabs.${v}`),
|
||||
})),
|
||||
@@ -778,6 +781,13 @@ export default function Channels() {
|
||||
onOpenWizard={onOpenWizard}
|
||||
onViewChannel={onViewChannel}
|
||||
/>
|
||||
) : view === "watched" ? (
|
||||
<WatchedChannels
|
||||
canWrite={canWrite}
|
||||
search={search}
|
||||
onOpenWizard={onOpenWizard}
|
||||
onViewChannel={onViewChannel}
|
||||
/>
|
||||
) : view === "blocked" ? (
|
||||
/* Blocked channels — videos from these are hidden everywhere and dropped from live search. */
|
||||
<div className="px-4 pb-4 pt-3 max-w-384 mx-auto">
|
||||
|
||||
@@ -48,7 +48,9 @@ import {
|
||||
// The contexts, their value types and the hooks live in channelsContext.ts.
|
||||
|
||||
function clampView(v: string | null): ChannelsView {
|
||||
return v === "discovery" || v === "blocked" || v === "hidden" ? v : "subscribed";
|
||||
return v === "discovery" || v === "watched" || v === "blocked" || v === "hidden"
|
||||
? v
|
||||
: "subscribed";
|
||||
}
|
||||
|
||||
// One-time carry-forward of the retired single-value sync chip (LS.legacyChannelFilter, vocab
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// The Channel manager's "Discover from your history" tab: what it says before a history was
|
||||
// uploaded and while the list is still being made, its top twenty and "show more", taking a channel
|
||||
// off the list with an Undo, and a row's count opening the channel's watched videos to play.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
|
||||
const h = vi.hoisted(() => ({
|
||||
notify: vi.fn(),
|
||||
confirm: vi.fn(async () => true),
|
||||
setPage: vi.fn(),
|
||||
focusSettingsTab: vi.fn(),
|
||||
player: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("react-i18next", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("react-i18next")>();
|
||||
return {
|
||||
...actual,
|
||||
useTranslation: () => ({
|
||||
i18n: { language: "en" },
|
||||
t: (key: string, o?: Record<string, unknown>) =>
|
||||
o && "count" in o ? `${key}:${String(o.count)}` : key,
|
||||
}),
|
||||
};
|
||||
});
|
||||
vi.mock("../../lib/notifications", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../../lib/notifications")>()),
|
||||
notify: (...a: unknown[]) => h.notify(...a),
|
||||
}));
|
||||
vi.mock("../../components/ui/confirmContext", () => ({ useConfirm: () => h.confirm }));
|
||||
vi.mock("../../features/navigation/navigationContext", () => ({
|
||||
useNavigationActions: () => ({ setPage: h.setPage }),
|
||||
}));
|
||||
vi.mock("../../lib/settingsTab", () => ({ focusSettingsTab: h.focusSettingsTab }));
|
||||
vi.mock("../../features/player/PlayerModal", () => ({
|
||||
default: (p: { video: { id: string }; queue: { id: string }[]; startIndex: number }) => {
|
||||
h.player(
|
||||
p.video.id,
|
||||
p.queue.map((v) => v.id),
|
||||
p.startIndex,
|
||||
);
|
||||
return <div data-testid="player" />;
|
||||
},
|
||||
}));
|
||||
|
||||
import WatchedChannels from "./WatchedChannels";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
import { api, type Video, type WatchedChannel, type WatchedChannels as List } from "../../lib/api";
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
for (const fn of Object.values(h)) fn.mockReset();
|
||||
h.confirm.mockImplementation(async () => true);
|
||||
});
|
||||
|
||||
const channel = (n: number, over: Partial<WatchedChannel> = {}): WatchedChannel => ({
|
||||
id: `UC${n}`,
|
||||
title: `Channel ${n}`,
|
||||
handle: null,
|
||||
thumbnail_url: null,
|
||||
subscriber_count: 1000,
|
||||
video_count: 50,
|
||||
description: null,
|
||||
watched: 30 - n,
|
||||
last_watched_at: "2026-09-24T10:00:00Z",
|
||||
...over,
|
||||
});
|
||||
|
||||
const video = (id: string): Video & { takeout_watched_at: string } => ({
|
||||
id,
|
||||
title: `Video ${id}`,
|
||||
channel_id: "UC1",
|
||||
channel_title: "Channel 1",
|
||||
channel_thumbnail: null,
|
||||
channel_url: "",
|
||||
published_at: null,
|
||||
thumbnail_url: null,
|
||||
duration_seconds: 600,
|
||||
view_count: null,
|
||||
is_short: false,
|
||||
live_status: "none",
|
||||
status: "new",
|
||||
position_seconds: 0,
|
||||
saved: false,
|
||||
watch_url: "",
|
||||
takeout_watched_at: "2026-09-24T10:00:00Z",
|
||||
});
|
||||
|
||||
function mount(list: List) {
|
||||
vi.spyOn(api, "watchedChannels").mockResolvedValue(list);
|
||||
const qc = new QueryClient({
|
||||
defaultOptions: { queries: { retry: false }, mutations: { retry: false } },
|
||||
});
|
||||
render(
|
||||
<QueryClientProvider client={qc}>
|
||||
<WatchedChannels canWrite search="" onOpenWizard={() => {}} onViewChannel={() => {}} />
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
return qc;
|
||||
}
|
||||
|
||||
describe("WatchedChannels", () => {
|
||||
it("sends a user with no history uploaded to the upload, not to an empty list", async () => {
|
||||
mount({ imported: false, pending: 0, channels: [] });
|
||||
fireEvent.click(await screen.findByRole("button", { name: "channels.watched.openTakeout" }));
|
||||
expect(h.focusSettingsTab).toHaveBeenCalledWith("account");
|
||||
expect(h.setPage).toHaveBeenCalledWith("settings");
|
||||
expect(screen.queryByText("channels.watched.empty")).toBeNull();
|
||||
});
|
||||
|
||||
it("says when more channels may still join, and when there is nothing to suggest", async () => {
|
||||
mount({ imported: true, pending: 3, channels: [] });
|
||||
expect(await screen.findByText("channels.watched.empty")).toBeTruthy();
|
||||
expect(screen.getByRole("status").textContent).toBe("channels.watched.pending:3");
|
||||
});
|
||||
|
||||
it("shows the top twenty, the most watched first, and twenty more on asking", async () => {
|
||||
mount({
|
||||
imported: true,
|
||||
pending: 0,
|
||||
channels: Array.from({ length: 27 }, (_, i) => channel(i)),
|
||||
});
|
||||
await screen.findByText("Channel 0");
|
||||
expect(screen.getByText("Channel 19")).toBeTruthy();
|
||||
expect(screen.queryByText("Channel 20")).toBeNull();
|
||||
fireEvent.click(screen.getByRole("button", { name: "channels.watched.more:7" }));
|
||||
expect(screen.getByText("Channel 26")).toBeTruthy();
|
||||
expect(screen.queryByRole("button", { name: /channels\.watched\.more/ })).toBeNull();
|
||||
});
|
||||
|
||||
it("takes a channel off the list with an Undo that puts it back", async () => {
|
||||
const dismiss = vi.spyOn(api, "dismissWatchedChannel").mockResolvedValue({ ok: true });
|
||||
const restore = vi.spyOn(api, "restoreWatchedChannel").mockResolvedValue({ ok: true });
|
||||
mount({ imported: true, pending: 0, channels: [channel(1)] });
|
||||
fireEvent.click(await screen.findByRole("button", { name: "channels.watched.dismiss" }));
|
||||
await waitFor(() => expect(h.notify).toHaveBeenCalled());
|
||||
expect(dismiss).toHaveBeenCalledWith("UC1");
|
||||
const toast = h.notify.mock.calls[0]![0] as { action: { label: string; onClick: () => void } };
|
||||
expect(toast.action.label).toBe("common.undo");
|
||||
act(() => toast.action.onClick());
|
||||
await waitFor(() => expect(restore).toHaveBeenCalledWith("UC1"));
|
||||
});
|
||||
|
||||
it("opens a row's watched videos from its count, plays one and plays them all", async () => {
|
||||
const videos = vi
|
||||
.spyOn(api, "watchedChannelVideos")
|
||||
.mockResolvedValue({ ok: true, items: [video("a"), video("b"), video("c")], gone: 2 });
|
||||
mount({ imported: true, pending: 0, channels: [channel(1)] });
|
||||
fireEvent.click(await screen.findByRole("button", { name: "29" }));
|
||||
const dialog = await screen.findByRole("dialog");
|
||||
await within(dialog).findByText("Video b");
|
||||
expect(videos).toHaveBeenCalledWith("UC1");
|
||||
expect(dialog.textContent).toContain("channels.watched.videos.gone:2");
|
||||
fireEvent.click(within(dialog).getByRole("button", { name: "card.play — Video b" }));
|
||||
await waitFor(() => expect(h.player).toHaveBeenLastCalledWith("b", ["a", "b", "c"], 1));
|
||||
fireEvent.click(within(dialog).getByRole("button", { name: "playlists.playAll" }));
|
||||
await waitFor(() => expect(h.player).toHaveBeenLastCalledWith("a", ["a", "b", "c"], 0));
|
||||
});
|
||||
|
||||
it("asks for a channel's videos once per opening, whatever the list does meanwhile", async () => {
|
||||
// The ask explores the channel and may spend quota: the list's own refetch (an Undo, a follow)
|
||||
// must not repeat it under an open dialog, nor swap the queue of a video playing.
|
||||
const videos = vi
|
||||
.spyOn(api, "watchedChannelVideos")
|
||||
.mockResolvedValue({ ok: true, items: [video("a")], gone: 0 });
|
||||
const qc = mount({ imported: true, pending: 0, channels: [channel(1)] });
|
||||
fireEvent.click(await screen.findByRole("button", { name: "29" }));
|
||||
await within(await screen.findByRole("dialog")).findByText("Video a");
|
||||
await act(() => qc.invalidateQueries({ queryKey: qk.watchedChannels() }));
|
||||
await waitFor(() => expect(api.watchedChannels).toHaveBeenCalledTimes(2));
|
||||
expect(videos).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("follows a channel from its videos after the confirm, and closes", async () => {
|
||||
vi.spyOn(api, "watchedChannelVideos").mockResolvedValue({
|
||||
ok: true,
|
||||
items: [video("a")],
|
||||
gone: 0,
|
||||
});
|
||||
const subscribe = vi.spyOn(api, "subscribeChannel").mockResolvedValue({ ok: true });
|
||||
mount({ imported: true, pending: 0, channels: [channel(1)] });
|
||||
fireEvent.click(await screen.findByRole("button", { name: "29" }));
|
||||
const dialog = await screen.findByRole("dialog");
|
||||
fireEvent.click(within(dialog).getByRole("button", { name: "channels.discovery.subscribe" }));
|
||||
await waitFor(() => expect(subscribe).toHaveBeenCalledWith("UC1"));
|
||||
expect(h.confirm).toHaveBeenCalled();
|
||||
await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull());
|
||||
});
|
||||
|
||||
it("follows nothing and stays open when the confirm is declined", async () => {
|
||||
vi.spyOn(api, "watchedChannelVideos").mockResolvedValue({
|
||||
ok: true,
|
||||
items: [video("a")],
|
||||
gone: 0,
|
||||
});
|
||||
const subscribe = vi.spyOn(api, "subscribeChannel").mockResolvedValue({ ok: true });
|
||||
h.confirm.mockImplementation(async () => false);
|
||||
mount({ imported: true, pending: 0, channels: [channel(1)] });
|
||||
fireEvent.click(await screen.findByRole("button", { name: "29" }));
|
||||
const dialog = await screen.findByRole("dialog");
|
||||
fireEvent.click(within(dialog).getByRole("button", { name: "channels.discovery.subscribe" }));
|
||||
await waitFor(() => expect(h.confirm).toHaveBeenCalled());
|
||||
expect(subscribe).not.toHaveBeenCalled();
|
||||
expect(screen.getByRole("dialog")).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,227 @@
|
||||
import { lazy, Suspense, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { EyeOff, History, UserPlus } from "lucide-react";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
import { api, type WatchedChannel } from "../../lib/api";
|
||||
import { formatDate } from "../../lib/format";
|
||||
import { focusSettingsTab } from "../../lib/settingsTab";
|
||||
import { useNavigationActions } from "../../features/navigation/navigationContext";
|
||||
import Tooltip from "../../components/ui/Tooltip";
|
||||
import DataTable, { type Column } from "../../components/ui/DataTable";
|
||||
import { LoadingState, StateMessage } from "../../components/ui/QueryState";
|
||||
import { Button } from "../../components/ui/Button";
|
||||
import ChannelLink from "./ChannelLink";
|
||||
import AboutCell from "./AboutCell";
|
||||
import ChannelLayoutGrid from "./ChannelLayoutGrid";
|
||||
import { subsColumn } from "./channelColumns";
|
||||
import { useChannelLayout } from "./channelsContext";
|
||||
import { useWatchedChannelActions } from "./useWatchedChannelActions";
|
||||
|
||||
const WatchedVideosDialog = lazy(() => import("./WatchedVideosDialog"));
|
||||
|
||||
// How many channels the list shows at first, and how many more each "Show more" adds. The list is a
|
||||
// ranking — the most watched first — so a short top is the point: past it, the reader asks for more.
|
||||
const WATCHED_PAGE = 20;
|
||||
|
||||
// The Channel manager's "Discover from your history" tab: the channels the user's Takeout watch
|
||||
// history shows them watching often but not following, the most watched first. A row's count opens
|
||||
// the videos it counts, ready to play — a quick look at what the channel was, before following it.
|
||||
export default function WatchedChannels({
|
||||
canWrite,
|
||||
search,
|
||||
onOpenWizard,
|
||||
onViewChannel,
|
||||
}: {
|
||||
canWrite: boolean;
|
||||
// Client-side name filter from the shared top SearchBar (matches title + handle).
|
||||
search: string;
|
||||
onOpenWizard: () => void;
|
||||
onViewChannel: (id: string, name: string) => void;
|
||||
}) {
|
||||
const { t, i18n } = useTranslation();
|
||||
const lang = i18n.language;
|
||||
const layout = useChannelLayout();
|
||||
const { setPage } = useNavigationActions();
|
||||
const [shown, setShown] = useState(WATCHED_PAGE);
|
||||
const [open, setOpen] = useState<WatchedChannel | null>(null);
|
||||
const actions = useWatchedChannelActions(onOpenWizard);
|
||||
|
||||
const query = useQuery({ queryKey: qk.watchedChannels(), queryFn: api.watchedChannels });
|
||||
|
||||
const q = search.trim().toLowerCase();
|
||||
const rows = (query.data?.channels ?? []).filter(
|
||||
(c) => !q || `${c.title ?? ""} ${c.handle ?? ""}`.toLowerCase().includes(q),
|
||||
);
|
||||
const visible = rows.slice(0, shown);
|
||||
|
||||
// The order is the ranking itself, so no column sorts: a header sort would only reorder the top
|
||||
// twenty and hide that it is a top.
|
||||
const columns: Column<WatchedChannel>[] = [
|
||||
{
|
||||
key: "actions",
|
||||
header: t("channels.cols.actions"),
|
||||
align: "left",
|
||||
nowrap: true,
|
||||
cardLabel: false,
|
||||
render: (c) => (
|
||||
<div className="flex items-center gap-1.5">
|
||||
<Tooltip
|
||||
always={!canWrite}
|
||||
hint={
|
||||
canWrite
|
||||
? t("channels.discovery.subscribeHint")
|
||||
: t("channels.discovery.needWriteHint")
|
||||
}
|
||||
>
|
||||
<Button
|
||||
variant="glass"
|
||||
size="xs"
|
||||
icon={<UserPlus />}
|
||||
onClick={() => void actions.onSubscribe(c)}
|
||||
aria-disabled={!canWrite}
|
||||
loading={actions.subscribing}
|
||||
>
|
||||
{t("channels.discovery.subscribe")}
|
||||
</Button>
|
||||
</Tooltip>
|
||||
<Tooltip hint={t("channels.watched.dismissHint")}>
|
||||
<Button variant="ghost" size="xs" icon={<EyeOff />} onClick={() => actions.dismiss(c)}>
|
||||
{t("channels.watched.dismiss")}
|
||||
</Button>
|
||||
</Tooltip>
|
||||
</div>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "channel",
|
||||
header: t("channels.cols.channel"),
|
||||
cardPrimary: true,
|
||||
render: (c) => (
|
||||
<div className="max-w-[18rem]">
|
||||
<ChannelLink
|
||||
id={c.id}
|
||||
title={c.title}
|
||||
handle={c.handle}
|
||||
thumbnailUrl={c.thumbnail_url}
|
||||
onView={() => onViewChannel(c.id, c.title ?? c.id)}
|
||||
/>
|
||||
</div>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "about",
|
||||
header: t("channels.cols.about"),
|
||||
hideInCard: true,
|
||||
render: (c) => <AboutCell text={c.description} channelId={c.id} />,
|
||||
},
|
||||
{ ...subsColumn<WatchedChannel>(t, lang), sortable: false },
|
||||
{
|
||||
key: "watched",
|
||||
header: t("channels.watched.cols.watched"),
|
||||
align: "right",
|
||||
nowrap: true,
|
||||
render: (c) => (
|
||||
<Tooltip hint={t("channels.watched.cols.watchedHint", { count: c.watched })}>
|
||||
<Button
|
||||
variant="link"
|
||||
size="xs"
|
||||
className="tabular-nums"
|
||||
icon={<History />}
|
||||
onClick={() => setOpen(c)}
|
||||
>
|
||||
{c.watched}
|
||||
</Button>
|
||||
</Tooltip>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "last",
|
||||
header: t("channels.watched.cols.last"),
|
||||
align: "right",
|
||||
nowrap: true,
|
||||
render: (c) => (
|
||||
<span className="text-muted tabular-nums">{formatDate(c.last_watched_at, lang)}</span>
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
const pending = query.data?.pending ?? 0;
|
||||
const more = Math.min(WATCHED_PAGE, rows.length - visible.length);
|
||||
const body =
|
||||
query.isError && !query.data ? (
|
||||
<StateMessage tone="error" onRetry={() => query.refetch()}>
|
||||
{t("common.loadError")}
|
||||
</StateMessage>
|
||||
) : query.isLoading ? (
|
||||
<LoadingState shape="list" label={t("channels.watched.loading")} />
|
||||
) : !query.data?.imported ? (
|
||||
// The message and the way to the upload, one line: StateMessage lays its content in a row.
|
||||
<StateMessage>
|
||||
{t("channels.watched.notImported")}
|
||||
<Button
|
||||
size="sm"
|
||||
icon={<History />}
|
||||
onClick={() => {
|
||||
focusSettingsTab("account");
|
||||
setPage("settings");
|
||||
}}
|
||||
>
|
||||
{t("channels.watched.openTakeout")}
|
||||
</Button>
|
||||
</StateMessage>
|
||||
) : layout === "table" ? (
|
||||
<DataTable
|
||||
rows={visible}
|
||||
columns={columns}
|
||||
rowKey={(c) => c.id}
|
||||
pager={false}
|
||||
emptyText={t("channels.watched.empty")}
|
||||
keyboardNav
|
||||
narrowedBy={q}
|
||||
/>
|
||||
) : (
|
||||
<ChannelLayoutGrid
|
||||
rows={visible}
|
||||
columns={columns}
|
||||
rowKey={(c) => c.id}
|
||||
aboutKey="about"
|
||||
actionsKey="actions"
|
||||
emptyText={t("channels.watched.empty")}
|
||||
/>
|
||||
);
|
||||
|
||||
return (
|
||||
<div className="px-4 pb-4 max-w-384 mx-auto">
|
||||
<p className="text-xs text-muted leading-relaxed pt-2">{t("channels.watched.intro")}</p>
|
||||
{pending > 0 && (
|
||||
<p className="text-xs text-muted leading-relaxed pt-1" role="status">
|
||||
{t("channels.watched.pending", { count: pending })}
|
||||
</p>
|
||||
)}
|
||||
{body}
|
||||
{rows.length > 0 && (
|
||||
<div className="flex items-center justify-between gap-3 pt-3 text-xs text-muted">
|
||||
<span className="tabular-nums">
|
||||
{t("channels.watched.shown", { shown: visible.length, total: rows.length })}
|
||||
</span>
|
||||
{more > 0 && (
|
||||
<Button size="sm" onClick={() => setShown((n) => n + WATCHED_PAGE)}>
|
||||
{t("channels.watched.more", { count: more })}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{open && (
|
||||
<Suspense fallback={null}>
|
||||
<WatchedVideosDialog
|
||||
channel={open}
|
||||
canWrite={canWrite}
|
||||
actions={actions}
|
||||
onClose={() => setOpen(null)}
|
||||
/>
|
||||
</Suspense>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
import { lazy, Suspense, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { EyeOff, Play, UserPlus } from "lucide-react";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
import { api, type Video, type VideoStatus, type WatchedChannel } from "../../lib/api";
|
||||
import { formatDate, formatDuration } from "../../lib/format";
|
||||
import { notifyActionError } from "../../lib/actionErrors";
|
||||
import { useStableCallback } from "../../lib/useLatest";
|
||||
import Modal, { ModalFooter } from "../../components/ui/Modal";
|
||||
import DataTable, { type Column } from "../../components/ui/DataTable";
|
||||
import { LoadingState, StateMessage } from "../../components/ui/QueryState";
|
||||
import { Button } from "../../components/ui/Button";
|
||||
import { Pressable } from "../../components/ui/Pressable";
|
||||
import RemoteImage from "../../components/ui/RemoteImage";
|
||||
import Tooltip from "../../components/ui/Tooltip";
|
||||
import type { WatchedChannelActions } from "./useWatchedChannelActions";
|
||||
|
||||
const PlayerModal = lazy(() => import("../../features/player/PlayerModal"));
|
||||
|
||||
type WatchedVideo = Video & { takeout_watched_at: string | null };
|
||||
|
||||
// The videos of one channel that the watched-channels list counts, last watched first, each ready to
|
||||
// play — and Play all through them, as on a playlist. Opening it explores the channel for the user
|
||||
// (the server brings in the videos the catalog lacks as the exploration's), so it is asked for once
|
||||
// per opening, not refetched behind the reader's back.
|
||||
export default function WatchedVideosDialog({
|
||||
channel,
|
||||
canWrite,
|
||||
actions,
|
||||
onClose,
|
||||
}: {
|
||||
channel: WatchedChannel;
|
||||
canWrite: boolean;
|
||||
actions: WatchedChannelActions;
|
||||
onClose: () => void;
|
||||
}) {
|
||||
const { t, i18n } = useTranslation();
|
||||
const lang = i18n.language;
|
||||
const qc = useQueryClient();
|
||||
const name = channel.title ?? channel.id;
|
||||
const key = qk.watchedChannelVideos(channel.id);
|
||||
const query = useQuery({
|
||||
queryKey: key,
|
||||
queryFn: () => api.watchedChannelVideos(channel.id),
|
||||
staleTime: Infinity,
|
||||
refetchOnWindowFocus: false,
|
||||
retry: false,
|
||||
});
|
||||
const items = query.data?.items ?? [];
|
||||
const gone = query.data?.gone ?? 0;
|
||||
const [playing, setPlaying] = useState<number | null>(null);
|
||||
|
||||
// The player's watched / in-progress marks: saved, then shown on this list and the feed's.
|
||||
const onState = useStableCallback((id: string, status: VideoStatus) => {
|
||||
api
|
||||
.setState(id, status)
|
||||
.then(() => {
|
||||
qc.setQueryData(
|
||||
key,
|
||||
(old: typeof query.data) =>
|
||||
old && {
|
||||
...old,
|
||||
items: old.items.map((v) => (v.id === id ? { ...v, status } : v)),
|
||||
},
|
||||
);
|
||||
void qc.invalidateQueries({ queryKey: qk.feed() });
|
||||
})
|
||||
.catch((e: unknown) => notifyActionError(e, t("errors.writeFailed.videoState")));
|
||||
});
|
||||
const closePlayer = useStableCallback(() => setPlaying(null));
|
||||
|
||||
const columns: Column<WatchedVideo>[] = [
|
||||
{
|
||||
key: "title",
|
||||
header: t("channels.watched.videos.colTitle"),
|
||||
width: "100%",
|
||||
cardPrimary: true,
|
||||
render: (v) => (
|
||||
<div className="flex items-center gap-2.5 min-w-0">
|
||||
<Pressable
|
||||
data-nav-open
|
||||
onClick={() => setPlaying(items.indexOf(v))}
|
||||
aria-label={`${t("card.play")} — ${v.title ?? v.id}`}
|
||||
className="relative w-20 h-11.25 rounded-sm overflow-hidden bg-surface shrink-0 group/thumb touch-target"
|
||||
>
|
||||
{v.thumbnail_url && (
|
||||
<RemoteImage src={v.thumbnail_url} alt="" className="w-full h-full object-cover" />
|
||||
)}
|
||||
<span className="absolute inset-0 grid place-items-center bg-black/40 opacity-0 group-hover/thumb:opacity-100 group-focus-visible/thumb:opacity-100 transition">
|
||||
<Play className="w-4 h-4 text-white fill-current" />
|
||||
</span>
|
||||
</Pressable>
|
||||
<span className="min-w-0 line-clamp-2 text-sm">{v.title ?? v.id}</span>
|
||||
</div>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "length",
|
||||
header: t("channels.watched.videos.colLength"),
|
||||
align: "right",
|
||||
nowrap: true,
|
||||
render: (v) => (
|
||||
<span className="text-muted tabular-nums">{formatDuration(v.duration_seconds)}</span>
|
||||
),
|
||||
},
|
||||
{
|
||||
key: "watched",
|
||||
header: t("channels.watched.videos.colWatched"),
|
||||
align: "right",
|
||||
nowrap: true,
|
||||
render: (v) => (
|
||||
<span className="text-muted tabular-nums">{formatDate(v.takeout_watched_at, lang)}</span>
|
||||
),
|
||||
},
|
||||
];
|
||||
|
||||
const body =
|
||||
query.isError && !query.data ? (
|
||||
<StateMessage tone="error" onRetry={() => query.refetch()}>
|
||||
{t("common.loadError")}
|
||||
</StateMessage>
|
||||
) : query.isLoading ? (
|
||||
<LoadingState shape="list" label={t("channels.watched.videos.loading")} />
|
||||
) : (
|
||||
<DataTable
|
||||
rows={items}
|
||||
columns={columns}
|
||||
rowKey={(v) => v.id}
|
||||
pager={false}
|
||||
emptyText={t("channels.watched.videos.empty")}
|
||||
keyboardNav
|
||||
/>
|
||||
);
|
||||
|
||||
return (
|
||||
<Modal
|
||||
title={t("channels.watched.videos.title", { name })}
|
||||
initialFocus="close"
|
||||
onClose={onClose}
|
||||
maxWidth="max-w-3xl"
|
||||
headerActions={
|
||||
<Button
|
||||
size="sm"
|
||||
icon={<Play className="fill-current" />}
|
||||
onClick={() => setPlaying(0)}
|
||||
disabled={!items.length}
|
||||
>
|
||||
{t("playlists.playAll")}
|
||||
</Button>
|
||||
}
|
||||
>
|
||||
<p className="text-xs text-muted pb-2">
|
||||
{t("channels.watched.videos.summary", {
|
||||
count: channel.watched,
|
||||
when: formatDate(channel.last_watched_at, lang),
|
||||
})}
|
||||
{gone > 0 && ` · ${t("channels.watched.videos.gone", { count: gone })}`}
|
||||
</p>
|
||||
{body}
|
||||
<ModalFooter>
|
||||
<Button
|
||||
variant="ghost"
|
||||
icon={<EyeOff />}
|
||||
onClick={() => {
|
||||
actions.dismiss(channel);
|
||||
onClose();
|
||||
}}
|
||||
>
|
||||
{t("channels.watched.dismiss")}
|
||||
</Button>
|
||||
<Tooltip
|
||||
always={!canWrite}
|
||||
hint={
|
||||
canWrite ? t("channels.discovery.subscribeHint") : t("channels.discovery.needWriteHint")
|
||||
}
|
||||
>
|
||||
<Button
|
||||
variant="outline-accent"
|
||||
icon={<UserPlus />}
|
||||
aria-disabled={!canWrite}
|
||||
loading={actions.subscribing}
|
||||
onClick={() => {
|
||||
void actions.onSubscribe(channel).then((ok) => ok && onClose());
|
||||
}}
|
||||
>
|
||||
{t("channels.discovery.subscribe")}
|
||||
</Button>
|
||||
</Tooltip>
|
||||
</ModalFooter>
|
||||
{playing != null && items[playing] && (
|
||||
<Suspense fallback={null}>
|
||||
<PlayerModal
|
||||
video={items[playing]}
|
||||
queue={items}
|
||||
startIndex={playing}
|
||||
startAt={null}
|
||||
onClose={closePlayer}
|
||||
onState={onState}
|
||||
/>
|
||||
</Suspense>
|
||||
)}
|
||||
</Modal>
|
||||
);
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import type { FilterMap } from "../../lib/tableColumns";
|
||||
// components (Fast Refresh). The provider owns the state; this is the read side every consumer imports.
|
||||
|
||||
// The manager's tabs.
|
||||
export type ChannelsView = "subscribed" | "discovery" | "hidden" | "blocked";
|
||||
export type ChannelsView = "subscribed" | "discovery" | "watched" | "hidden" | "blocked";
|
||||
|
||||
export type ChannelsState = {
|
||||
search: string;
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
import { invalidateSubscription } from "../../lib/subscriptionCache";
|
||||
import { api, type WatchedChannel } from "../../lib/api";
|
||||
import { notify } from "../../lib/notifications";
|
||||
import { notifyYouTubeActionError } from "../../lib/youtubeErrors";
|
||||
import { useConfirm } from "../../components/ui/confirmContext";
|
||||
|
||||
/** Follow and take-off-the-list for a watched channel: the tab's rows and its videos dialog share
|
||||
* them, so the confirm, the toasts and the cache refresh are written once. */
|
||||
export function useWatchedChannelActions(onOpenWizard: () => void) {
|
||||
const { t } = useTranslation();
|
||||
const qc = useQueryClient();
|
||||
const confirm = useConfirm();
|
||||
const nameOf = (c: WatchedChannel) => c.title ?? c.id;
|
||||
|
||||
const subscribe = useMutation({
|
||||
mutationFn: (c: WatchedChannel) => api.subscribeChannel(c.id),
|
||||
onSuccess: (_data, c) => {
|
||||
invalidateSubscription(qc, c.id);
|
||||
notify({
|
||||
level: "success",
|
||||
title: t("channels.discovery.subscribedTitle"),
|
||||
message: t("channels.discovery.subscribedBody", { name: nameOf(c) }),
|
||||
meta: { kind: "channel-subscribed", channelId: c.id, channelName: nameOf(c) },
|
||||
});
|
||||
},
|
||||
onError: (err: unknown) =>
|
||||
notifyYouTubeActionError(err, t("channels.discovery.subscribeFailed"), onOpenWizard),
|
||||
});
|
||||
const restore = useMutation({
|
||||
mutationFn: (c: WatchedChannel) => api.restoreWatchedChannel(c.id),
|
||||
meta: { failTitle: "errors.writeFailed.restoreWatchedChannel" },
|
||||
onSuccess: () => qc.invalidateQueries({ queryKey: qk.watchedChannels() }),
|
||||
});
|
||||
// Taken off the list at once, with the toast's Undo as the way back (the same bargain as hiding a
|
||||
// channel): the next channel steps up into its place.
|
||||
const dismiss = useMutation({
|
||||
mutationFn: (c: WatchedChannel) => api.dismissWatchedChannel(c.id),
|
||||
meta: { failTitle: "errors.writeFailed.dismissWatchedChannel" },
|
||||
onSuccess: (_data, c) => {
|
||||
void qc.invalidateQueries({ queryKey: qk.watchedChannels() });
|
||||
notify({
|
||||
level: "success",
|
||||
message: t("channels.watched.dismissed", { name: nameOf(c) }),
|
||||
action: { label: t("common.undo"), onClick: () => restore.mutate(c) },
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
// Subscribing changes the user's real YouTube account and spends a little quota — confirmed, as
|
||||
// on the Discovery tab. Resolves whether it went ahead.
|
||||
const onSubscribe = async (c: WatchedChannel): Promise<boolean> => {
|
||||
const ok = await confirm({
|
||||
title: t("channels.discovery.subscribe"),
|
||||
message: t("channels.discovery.confirmSubscribe", { name: nameOf(c) }),
|
||||
confirmLabel: t("channels.discovery.subscribe"),
|
||||
});
|
||||
if (ok) subscribe.mutate(c);
|
||||
return ok;
|
||||
};
|
||||
return {
|
||||
onSubscribe,
|
||||
subscribing: subscribe.isPending,
|
||||
dismiss: (c: WatchedChannel) => dismiss.mutate(c),
|
||||
};
|
||||
}
|
||||
|
||||
export type WatchedChannelActions = ReturnType<typeof useWatchedChannelActions>;
|
||||
@@ -157,6 +157,8 @@ export default function ChannelPage({
|
||||
qc.invalidateQueries({ queryKey: qk.channel(channelId) });
|
||||
qc.invalidateQueries({ queryKey: qk.feed() });
|
||||
qc.invalidateQueries({ queryKey: qk.blockedChannels() });
|
||||
// A blocked channel is off the watched-channels list; unblocked, it may be back on it.
|
||||
qc.invalidateQueries({ queryKey: qk.watchedChannels() });
|
||||
},
|
||||
// No onError: a local DB op (not a YouTube write, so no 403 "connect" case). The app-wide
|
||||
// reporter names the failure the error dialog does not cover (a 404 for a channel gone).
|
||||
|
||||
@@ -15,7 +15,7 @@ import type { DownloadJob } from "../../lib/api";
|
||||
import { useFilesActions } from "./filesContext";
|
||||
import { formatBytes } from "../../lib/format";
|
||||
import { DISPLAY_NAME_MAX } from "../../lib/api/downloads";
|
||||
import { Pressable } from "../../components/ui/Pressable";
|
||||
import { DropZone } from "../../components/ui/DropZone";
|
||||
import { Input } from "../../components/ui/Input";
|
||||
|
||||
// Local-file ingest. A general upload — ANY file type, no format list: media
|
||||
@@ -46,13 +46,8 @@ export default function UploadDialog({
|
||||
const qc = useQueryClient();
|
||||
const [file, setFile] = useState<File | null>(initialFile);
|
||||
const [name, setName] = useState("");
|
||||
const [dragOver, setDragOver] = useState(false);
|
||||
const [pct, setPct] = useState<number | null>(null); // null = idle; 0–100 while uploading
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const abortRef = useRef<AbortController | null>(null);
|
||||
// dragenter/dragleave fire as the cursor crosses the drop zone's OWN children (icon/text), so a
|
||||
// plain boolean flickers. Count enter vs leave and only clear the highlight at depth 0.
|
||||
const dragDepth = useRef(0);
|
||||
const busy = pct !== null;
|
||||
// The per-file cap, so an over-cap file is refused HERE instead of after the user has pushed it up
|
||||
// the wire. Both server-side caps are still the boundary — this is only the pleasant half of it.
|
||||
@@ -86,8 +81,6 @@ export default function UploadDialog({
|
||||
// footer Cancel wired the abort; this covers every other dismissal.
|
||||
useEffect(() => () => abortRef.current?.abort(), []);
|
||||
|
||||
const pick = () => inputRef.current?.click();
|
||||
|
||||
const submit = async () => {
|
||||
// `tooBig` here is belt and braces, NOT the mechanism: `ui/Button` is focus-safe, so while the
|
||||
// button holds focus it swallows the activation itself, and while it does not it is natively
|
||||
@@ -138,74 +131,33 @@ export default function UploadDialog({
|
||||
<Modal initialFocus="field" title={t("downloads.upload.title")} onClose={onClose}>
|
||||
<p className="text-sm text-muted mb-4">{t("downloads.upload.hint")}</p>
|
||||
|
||||
{/* Drop zone doubles as the picker trigger — a click anywhere opens the native file dialog. */}
|
||||
<Pressable
|
||||
align="center"
|
||||
onClick={pick}
|
||||
onDragEnter={(e) => {
|
||||
e.preventDefault();
|
||||
dragDepth.current += 1;
|
||||
setDragOver(true);
|
||||
}}
|
||||
onDragOver={(e) => e.preventDefault()}
|
||||
onDragLeave={() => {
|
||||
dragDepth.current = Math.max(0, dragDepth.current - 1);
|
||||
if (dragDepth.current === 0) setDragOver(false);
|
||||
}}
|
||||
onDrop={(e) => {
|
||||
e.preventDefault();
|
||||
dragDepth.current = 0;
|
||||
setDragOver(false);
|
||||
const f = e.dataTransfer.files?.[0];
|
||||
if (f) pick_(f);
|
||||
}}
|
||||
{/* Drop zone doubles as the picker trigger — a click anywhere opens the native file dialog.
|
||||
With a file picked, its label shows the file's name rather than `dropActive`, so the
|
||||
zone's drag border is the only feedback that it will take a replacement. */}
|
||||
<DropZone
|
||||
className="px-4 py-8"
|
||||
onFiles={([f]) => pick_(f)}
|
||||
disabled={busy}
|
||||
// The refusal is tied to the control it invalidates, not left to a one-shot announcement:
|
||||
// the drop zone reads as invalid and points at the sentence, so a reader who tabs back to
|
||||
// it hears WHY. (Measured, because `aria-invalid` is not in `button`'s ARIA-supported
|
||||
// list: Chromium's accessibility node for a native button does carry `invalid` from this
|
||||
// attribute.) The signal that is NOT colour is the sentence below, visible to everyone —
|
||||
// the border only reinforces it.
|
||||
aria-invalid={tooBig}
|
||||
aria-describedby={tooBig ? refusalId : undefined}
|
||||
// `dragOver` outranks `tooBig`: a drag is transient and the refusal repaints the instant
|
||||
// the drop lands, whereas a zone that stays red under the pointer tells the user their
|
||||
// drop target is dead — in the one state where they are trying to replace the file. With
|
||||
// a file picked, the label below shows its name rather than `dropActive`, so this border
|
||||
// is the ONLY feedback that the zone will accept the drop.
|
||||
className={`w-full flex flex-col items-center justify-center gap-2 rounded-xl border-2 border-dashed px-4 py-8 text-center transition disabled:opacity-60 ${
|
||||
dragOver
|
||||
? "border-accent bg-accent/5"
|
||||
: tooBig
|
||||
? "border-danger"
|
||||
: "border-border hover:border-fg/40"
|
||||
}`}
|
||||
invalid={tooBig}
|
||||
describedBy={tooBig ? refusalId : undefined}
|
||||
>
|
||||
{file ? (
|
||||
<>
|
||||
<FileIcon className="w-6 h-6 text-muted" />
|
||||
<span className="text-sm font-medium break-all">{file.name}</span>
|
||||
<span className="text-xs text-muted">{formatBytes(file.size, lang)}</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<UploadCloud className="w-7 h-7 text-muted" />
|
||||
<span className="text-sm text-muted">
|
||||
{dragOver ? t("downloads.upload.dropActive") : t("downloads.upload.drop")}
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</Pressable>
|
||||
<input
|
||||
ref={inputRef}
|
||||
type="file"
|
||||
className="hidden"
|
||||
onChange={(e) => {
|
||||
const f = e.target.files?.[0];
|
||||
if (f) pick_(f);
|
||||
e.target.value = ""; // let re-picking the same file fire onChange again
|
||||
}}
|
||||
/>
|
||||
{(dragOver) =>
|
||||
file ? (
|
||||
<>
|
||||
<FileIcon className="w-6 h-6 text-muted" />
|
||||
<span className="text-sm font-medium break-all">{file.name}</span>
|
||||
<span className="text-xs text-muted">{formatBytes(file.size, lang)}</span>
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<UploadCloud className="w-7 h-7 text-muted" />
|
||||
<span className="text-sm text-muted">
|
||||
{dragOver ? t("downloads.upload.dropActive") : t("downloads.upload.drop")}
|
||||
</span>
|
||||
</>
|
||||
)
|
||||
}
|
||||
</DropZone>
|
||||
|
||||
{tooBig && (
|
||||
<p key={pickSeq} id={refusalId} className="text-xs text-danger mt-2" role="alert">
|
||||
|
||||
@@ -205,6 +205,34 @@ describe("NotificationsPanel · a server row", () => {
|
||||
expect(menuRows()).toEqual(["inbox.openTokens", "inbox.markRead", "inbox.dismiss"]);
|
||||
});
|
||||
|
||||
it("a Takeout import's run opens the user's own import, any other run the Scheduler", async () => {
|
||||
const run = (id: number, job: string) =>
|
||||
server(id, {
|
||||
type: "scheduler",
|
||||
data: { job_id: job, status: "ok", summary: `${job} done` },
|
||||
});
|
||||
h.server = [run(1, "takeout_import"), run(2, "channel_backfill")];
|
||||
h.client = [];
|
||||
mount();
|
||||
fireEvent.click(screen.getByText("inbox.openTakeout"));
|
||||
expect(h.focusSettingsTab).toHaveBeenCalledWith("account");
|
||||
expect(h.setPage).toHaveBeenLastCalledWith("settings");
|
||||
fireEvent.click(screen.getByText("inbox.openScheduler"));
|
||||
expect(h.setPage).toHaveBeenLastCalledWith("scheduler");
|
||||
const takeoutRow = screen
|
||||
.getByText("inbox.openTakeout")
|
||||
.closest<HTMLElement>("[data-nav-index]")!;
|
||||
await openMenu(() =>
|
||||
fireEvent.click(takeoutRow.querySelector<HTMLElement>('button[aria-label="inbox.rowMenu"]')!),
|
||||
);
|
||||
expect(menuRows()).toEqual(["inbox.openTakeout", "inbox.markRead", "inbox.dismiss"]);
|
||||
h.focusSettingsTab.mockClear();
|
||||
h.setPage.mockClear();
|
||||
choose("inbox.openTakeout");
|
||||
await waitFor(() => expect(h.focusSettingsTab).toHaveBeenCalledWith("account"));
|
||||
expect(h.setPage).toHaveBeenCalledWith("settings");
|
||||
});
|
||||
|
||||
it("holds the link, mark read and dismiss", async () => {
|
||||
h.server = [server(1)];
|
||||
h.client = [];
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
ExternalLink,
|
||||
Eye,
|
||||
FolderOpen,
|
||||
History,
|
||||
KeyRound,
|
||||
RotateCcw,
|
||||
Search,
|
||||
@@ -19,7 +20,7 @@ import {
|
||||
UserPlus,
|
||||
X,
|
||||
} from "lucide-react";
|
||||
import { api, type AppNotification } from "../../lib/api";
|
||||
import { api, TAKEOUT_JOB, type AppNotification } from "../../lib/api";
|
||||
import { jobRunText } from "../../lib/jobResult";
|
||||
import { notifyActionError } from "../../lib/actionErrors";
|
||||
import { useLiveQuery } from "../../lib/useLiveQuery";
|
||||
@@ -192,6 +193,10 @@ export default function NotificationsPanel() {
|
||||
focusSettingsTab("tokens");
|
||||
setPage("settings");
|
||||
};
|
||||
const openTakeout = () => {
|
||||
focusSettingsTab("account");
|
||||
setPage("settings");
|
||||
};
|
||||
// A finished download opens on its file, in its folder; a notice from before it named one, on Files.
|
||||
const openDone = (n: AppNotification) => {
|
||||
const { job_id: id, folder_id: folder } = n.data ?? {};
|
||||
@@ -213,34 +218,43 @@ export default function NotificationsPanel() {
|
||||
const kind = serverKind(n);
|
||||
return menuGroups(
|
||||
[
|
||||
...(kind === "scheduler"
|
||||
...(kind === "takeout"
|
||||
? [
|
||||
{
|
||||
id: "open",
|
||||
label: t("inbox.openScheduler"),
|
||||
icon: icon(Activity),
|
||||
onSelect: openScheduler,
|
||||
label: t("inbox.openTakeout"),
|
||||
icon: icon(History),
|
||||
onSelect: openTakeout,
|
||||
},
|
||||
]
|
||||
: kind === "download"
|
||||
: kind === "scheduler"
|
||||
? [
|
||||
{
|
||||
id: "open",
|
||||
label: t("inbox.openFile"),
|
||||
icon: icon(FolderOpen),
|
||||
onSelect: () => openDone(n),
|
||||
label: t("inbox.openScheduler"),
|
||||
icon: icon(Activity),
|
||||
onSelect: openScheduler,
|
||||
},
|
||||
]
|
||||
: kind === "token"
|
||||
: kind === "download"
|
||||
? [
|
||||
{
|
||||
id: "open",
|
||||
label: t("inbox.openTokens"),
|
||||
icon: icon(KeyRound),
|
||||
onSelect: openTokens,
|
||||
label: t("inbox.openFile"),
|
||||
icon: icon(FolderOpen),
|
||||
onSelect: () => openDone(n),
|
||||
},
|
||||
]
|
||||
: []),
|
||||
: kind === "token"
|
||||
? [
|
||||
{
|
||||
id: "open",
|
||||
label: t("inbox.openTokens"),
|
||||
icon: icon(KeyRound),
|
||||
onSelect: openTokens,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(!n.read
|
||||
? [
|
||||
{
|
||||
@@ -411,6 +425,7 @@ export default function NotificationsPanel() {
|
||||
onRead={() => readMut.mutate(n.id)}
|
||||
onDismiss={() => dismissMut.mutate(n.id)}
|
||||
onOpenScheduler={openScheduler}
|
||||
onOpenTakeout={openTakeout}
|
||||
onOpenFile={() => openDone(n)}
|
||||
onOpenTokens={openTokens}
|
||||
/>
|
||||
@@ -478,7 +493,9 @@ const DONE_TITLE: Record<string, string | undefined> = {
|
||||
upload: "inbox.uploadDone.title",
|
||||
};
|
||||
|
||||
function serverKind(n: AppNotification): "scheduler" | "download" | "token" | null {
|
||||
function serverKind(n: AppNotification): "scheduler" | "takeout" | "download" | "token" | null {
|
||||
// A Takeout import's run is the user's own: told like any run, opened where they started it.
|
||||
if (n.type === "scheduler" && n.data?.job_id === TAKEOUT_JOB) return "takeout";
|
||||
if (n.type === "scheduler" && typeof n.data?.job_id === "string") return "scheduler";
|
||||
if (n.type === "download_done") return "download";
|
||||
if (n.type === "api_token_expiring" && typeof n.data?.name === "string") return "token";
|
||||
@@ -527,6 +544,7 @@ function NotificationRow({
|
||||
onRead,
|
||||
onDismiss,
|
||||
onOpenScheduler,
|
||||
onOpenTakeout,
|
||||
onOpenFile,
|
||||
onOpenTokens,
|
||||
}: {
|
||||
@@ -538,6 +556,7 @@ function NotificationRow({
|
||||
onRead: () => void;
|
||||
onDismiss: () => void;
|
||||
onOpenScheduler: () => void;
|
||||
onOpenTakeout: () => void;
|
||||
onOpenFile: () => void;
|
||||
onOpenTokens: () => void;
|
||||
}) {
|
||||
@@ -549,7 +568,8 @@ function NotificationRow({
|
||||
// Known notification types are rendered from i18n (so they're trilingual) using the typed
|
||||
// payload; the server-stored title/body are an English fallback for any unknown type.
|
||||
const isMaintenance = n.type === "maintenance" && typeof n.data?.count === "number";
|
||||
const isScheduler = serverKind(n) === "scheduler";
|
||||
const isTakeout = serverKind(n) === "takeout";
|
||||
const isScheduler = serverKind(n) === "scheduler" || isTakeout;
|
||||
const isDownloadDone = serverKind(n) === "download";
|
||||
const isTokenExpiring = serverKind(n) === "token";
|
||||
const isEmailUndelivered = n.type === "email_undelivered" && typeof n.data?.reason === "string";
|
||||
@@ -633,10 +653,12 @@ function NotificationRow({
|
||||
variant="link"
|
||||
className="text-sm mt-1.5"
|
||||
data-nav-open=""
|
||||
onClick={onOpenScheduler}
|
||||
icon={<Activity className="w-3.5 h-3.5" />}
|
||||
onClick={isTakeout ? onOpenTakeout : onOpenScheduler}
|
||||
icon={
|
||||
isTakeout ? <History className="w-3.5 h-3.5" /> : <Activity className="w-3.5 h-3.5" />
|
||||
}
|
||||
>
|
||||
{t("inbox.openScheduler")}
|
||||
{t(isTakeout ? "inbox.openTakeout" : "inbox.openScheduler")}
|
||||
</Button>
|
||||
)}
|
||||
{isTokenExpiring && (
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
import { DownloadFormats } from "./ProfileEditor";
|
||||
import DiagnosticsSettings from "./DiagnosticsSettings";
|
||||
import ApiTokens from "./ApiTokens";
|
||||
import TakeoutImport from "./TakeoutImport";
|
||||
import { type SettingsTabId } from "../../lib/settingsTab";
|
||||
import { SCHEMES, type Scheme } from "../../lib/theme";
|
||||
import { api, HttpError, type Me } from "../../lib/api";
|
||||
@@ -870,6 +871,8 @@ function Account({ me, onOpenWizard }: { me: Me; onOpenWizard: () => void }) {
|
||||
</Section>
|
||||
) : null}
|
||||
|
||||
<TakeoutImport />
|
||||
|
||||
<Section title={t("settings.account.dangerZone")}>
|
||||
<p className="text-xs text-muted leading-relaxed mb-2">
|
||||
{t("settings.account.deleteHint")}
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// Settings › Account › YouTube history: the Google Takeout files are dropped (or picked) and sent one
|
||||
// request each, every file's outcome said on its own; the section then says what the import did —
|
||||
// only the counts it produced, history first — or that it is running, failed or was cut off. Before
|
||||
// the first import the drop zone is open; after one, "New import" opens it. i18n is key-passthrough
|
||||
// (a key's options ride along in brackets).
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
|
||||
import { api, HttpError, type TakeoutStatus, type TakeoutUpload } from "../../lib/api";
|
||||
|
||||
const mocks = vi.hoisted(() => ({ notify: vi.fn(), notifyActionError: vi.fn() }));
|
||||
vi.mock("react-i18next", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("react-i18next")>()),
|
||||
useTranslation: () => ({
|
||||
i18n: { language: "en" },
|
||||
t: (key: string, o?: Record<string, unknown>) =>
|
||||
o && Object.keys(o).length ? `${key}(${Object.values(o).join(",")})` : key,
|
||||
}),
|
||||
}));
|
||||
vi.mock("../../lib/notifications", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../../lib/notifications")>()),
|
||||
notify: mocks.notify,
|
||||
}));
|
||||
vi.mock("../../lib/actionErrors", () => ({ notifyActionError: mocks.notifyActionError }));
|
||||
vi.mock("../../lib/format", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../../lib/format")>()),
|
||||
formatDateTime: (iso: string) => `at ${iso}`,
|
||||
}));
|
||||
|
||||
import TakeoutImport from "./TakeoutImport";
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.restoreAllMocks();
|
||||
mocks.notify.mockClear();
|
||||
mocks.notifyActionError.mockClear();
|
||||
});
|
||||
|
||||
const run = (status: string, extra: Partial<NonNullable<TakeoutStatus["run"]>> = {}) => ({
|
||||
status: status as NonNullable<TakeoutStatus["run"]>["status"],
|
||||
started_at: "S",
|
||||
finished_at: status === "running" || status === "interrupted" ? null : "F",
|
||||
result: null,
|
||||
failure: null,
|
||||
...extra,
|
||||
});
|
||||
const uploaded = (videos: number): TakeoutUpload => ({
|
||||
ok: true,
|
||||
kind: "watched",
|
||||
videos,
|
||||
entries: videos,
|
||||
ads: 0,
|
||||
other: 0,
|
||||
malformed: 0,
|
||||
});
|
||||
|
||||
// A whole status: nothing queued and room for any file, unless the test says otherwise.
|
||||
const st = (s: Partial<TakeoutStatus>): TakeoutStatus => ({
|
||||
queued: false,
|
||||
max_bytes: 1_000_000,
|
||||
run: null,
|
||||
...s,
|
||||
});
|
||||
|
||||
function mount(status: TakeoutStatus | Error | Promise<never>) {
|
||||
const spy = vi.spyOn(api, "takeoutStatus");
|
||||
if (status instanceof Error) spy.mockRejectedValue(status);
|
||||
else if (status instanceof Promise) spy.mockReturnValue(status);
|
||||
else spy.mockResolvedValue(status);
|
||||
render(
|
||||
<QueryClientProvider
|
||||
client={new QueryClient({ defaultOptions: { queries: { retry: false } } })}
|
||||
>
|
||||
<TakeoutImport />
|
||||
</QueryClientProvider>,
|
||||
);
|
||||
return spy;
|
||||
}
|
||||
|
||||
const zone = () => screen.queryByRole("button", { name: /settings\.takeout\.drop/ });
|
||||
const line = () => screen.getByTestId("takeout-last").textContent;
|
||||
const drop = (files: File[]) =>
|
||||
act(async () => {
|
||||
fireEvent.drop(zone()!, { dataTransfer: { files } });
|
||||
});
|
||||
const file = (name: string, size = 2) => new File(["x".repeat(size)], name);
|
||||
const unavailable = (el: HTMLElement | null) =>
|
||||
!!el && ((el as HTMLButtonElement).disabled || el.getAttribute("aria-disabled") === "true");
|
||||
|
||||
describe("YouTube history import · what it says", () => {
|
||||
it("before the first import: the drop zone, and no line about an import", async () => {
|
||||
mount(st({ run: null, watched: {}, watch_later: {} }));
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
expect(screen.queryByTestId("takeout-last")).toBeNull();
|
||||
});
|
||||
|
||||
it("not while the last import is still being asked for", () => {
|
||||
mount(new Promise<never>(() => {}));
|
||||
expect(zone()).toBeNull();
|
||||
});
|
||||
|
||||
it("after one: when, then only the counts it produced, the history's first", async () => {
|
||||
mount(
|
||||
st({
|
||||
run: run("ok"),
|
||||
watched: { watched: 1020, waiting: 6785, short: 0, kept: 81 },
|
||||
watch_later: { added: 41, waiting: 2 },
|
||||
}),
|
||||
);
|
||||
await waitFor(() => expect(screen.getByTestId("takeout-last")).toBeTruthy());
|
||||
expect(line()).toBe(
|
||||
[
|
||||
"settings.takeout.last(at F)",
|
||||
"settings.takeout.count.watched(1020,1,020)",
|
||||
"settings.takeout.count.waiting(6785,6,785)",
|
||||
"settings.takeout.count.kept(81,81)",
|
||||
"settings.takeout.count.added(41,41)",
|
||||
"settings.takeout.count.laterWaiting(2,2)",
|
||||
].join(" · "),
|
||||
);
|
||||
expect(zone()).toBeNull();
|
||||
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
|
||||
expect(zone()).toBeTruthy();
|
||||
});
|
||||
|
||||
it("a failed run and one a restart cut off say so, not counts", async () => {
|
||||
mount(st({ run: run("error"), watched: { watched: 3 } }));
|
||||
await waitFor(() => expect(line()).toBe("settings.takeout.failed(at F)"));
|
||||
cleanup();
|
||||
mount(st({ run: run("interrupted"), watched: { watched: 3 } }));
|
||||
await waitFor(() => expect(line()).toBe("settings.takeout.interrupted(at S)"));
|
||||
});
|
||||
|
||||
it("a running import: the running line, the zone unavailable, asked again until it ends", async () => {
|
||||
const spy = mount(st({ run: run("running"), watched: {} }));
|
||||
await waitFor(() =>
|
||||
expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"),
|
||||
);
|
||||
expect(unavailable(zone())).toBe(true);
|
||||
spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 5 } }));
|
||||
await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(5,5)"), {
|
||||
timeout: 3000,
|
||||
});
|
||||
});
|
||||
|
||||
it("a status that could not be read says so, with a retry — not an empty section", async () => {
|
||||
mount(new HttpError(500, "boom"));
|
||||
await waitFor(() =>
|
||||
expect(screen.getByRole("alert").textContent).toContain("settings.takeout.loadFailed"),
|
||||
);
|
||||
expect(zone()).toBeNull();
|
||||
expect(screen.queryByTestId("takeout-last")).toBeNull();
|
||||
});
|
||||
|
||||
it("a re-read that fails does not leave the previous answer standing under it", async () => {
|
||||
const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("OLD"));
|
||||
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
|
||||
vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
|
||||
status.mockRejectedValue(new HttpError(500, "boom"));
|
||||
await drop([file("watch-history.json")]);
|
||||
await waitFor(() =>
|
||||
expect(screen.getByRole("alert").textContent).toContain("settings.takeout.loadFailed"),
|
||||
);
|
||||
expect(screen.queryByTestId("takeout-last")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("YouTube history import · the upload", () => {
|
||||
it("sends each dropped file on its own, says each outcome, and reads the new run", async () => {
|
||||
const status = mount(st({ run: null, watched: {} }));
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
const refused = new HttpError(422, "refused");
|
||||
const sent: string[] = [];
|
||||
vi.spyOn(api, "takeoutUpload").mockImplementation(async (f: File) => {
|
||||
sent.push(f.name);
|
||||
if (f.name === "bad.html") throw refused;
|
||||
return uploaded(f.name === "watch-history.json" ? 9034 : 56);
|
||||
});
|
||||
status.mockResolvedValue(st({ run: run("ok"), watched: { watched: 7 } }));
|
||||
await drop([file("watch-history.json"), file("bad.html"), file("Watch later-videos.csv")]);
|
||||
|
||||
await waitFor(() => expect(line()).toContain("settings.takeout.count.watched(7,7)"));
|
||||
expect(sent).toEqual(["watch-history.json", "bad.html", "Watch later-videos.csv"]);
|
||||
expect(mocks.notify.mock.calls.map(([n]) => (n as { message: string }).message)).toEqual([
|
||||
"settings.takeout.read(watch-history.json,9034,9,034)",
|
||||
"settings.takeout.read(Watch later-videos.csv,56,56)",
|
||||
]);
|
||||
expect(mocks.notifyActionError).toHaveBeenCalledWith(
|
||||
refused,
|
||||
"errors.writeFailed.takeoutUpload",
|
||||
);
|
||||
});
|
||||
|
||||
it("names the file being sent, then never shows the previous import's line before the new one", async () => {
|
||||
const status = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("OLD"));
|
||||
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
|
||||
let finish!: (v: TakeoutUpload) => void;
|
||||
vi.spyOn(api, "takeoutUpload").mockReturnValue(new Promise((r) => (finish = r)));
|
||||
let answer!: (v: TakeoutStatus) => void;
|
||||
status.mockReturnValue(new Promise((r) => (answer = r)));
|
||||
await drop([file("watch-history.json")]);
|
||||
expect(screen.getByTestId("takeout-running").textContent).toBe(
|
||||
"settings.takeout.uploading(watch-history.json)",
|
||||
);
|
||||
await act(async () => finish(uploaded(1)));
|
||||
// The new run is still being read: the upload's line stays, the old import's does not return.
|
||||
expect(screen.queryByTestId("takeout-last")).toBeNull();
|
||||
await act(async () => answer(st({ run: run("ok", { finished_at: "NEW" }), watched: {} })));
|
||||
await waitFor(() => expect(line()).toContain("NEW"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("YouTube history import · while it runs", () => {
|
||||
it("a queued import is this upload's, not the last one: the running line until its run reads", async () => {
|
||||
const spy = mount(st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} }));
|
||||
await waitFor(() =>
|
||||
expect(screen.getByTestId("takeout-running").textContent).toBe("settings.takeout.running"),
|
||||
);
|
||||
expect(screen.queryByTestId("takeout-last")).toBeNull();
|
||||
spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 });
|
||||
});
|
||||
|
||||
it("a re-read that fails mid-run does not leave the spinner claiming progress under it", async () => {
|
||||
const spy = mount(st({ run: run("running"), watched: {} }));
|
||||
await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy());
|
||||
spy.mockRejectedValue(new HttpError(500, "boom"));
|
||||
await waitFor(
|
||||
() => expect(screen.getByRole("alert").textContent).toContain("settings.takeout.loadFailed"),
|
||||
{ timeout: 3000 },
|
||||
);
|
||||
expect(screen.queryByTestId("takeout-running")).toBeNull();
|
||||
spy.mockResolvedValue(st({ run: run("ok"), watched: {} }));
|
||||
await waitFor(() => expect(screen.queryByRole("alert")).toBeNull(), { timeout: 3000 });
|
||||
});
|
||||
|
||||
it("one live region says the run and then its outcome", async () => {
|
||||
const spy = mount(st({ run: run("running"), watched: {} }));
|
||||
await waitFor(() => expect(screen.getByTestId("takeout-running")).toBeTruthy());
|
||||
const region = screen.getByRole("status");
|
||||
spy.mockResolvedValue(st({ run: run("ok"), watched: { watched: 2 } }));
|
||||
await waitFor(() => expect(line()).toContain("watched(2,2)"), { timeout: 3000 });
|
||||
expect(screen.getByRole("status")).toBe(region);
|
||||
expect(region.contains(screen.getByTestId("takeout-last"))).toBe(true);
|
||||
});
|
||||
|
||||
it("a file dropped beside the zone is kept from the browser, which would open it", async () => {
|
||||
mount(st({ run: null, watched: {} }));
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
const missed = (type: string, types: string[]) => {
|
||||
const e = new Event(type, { bubbles: true, cancelable: true });
|
||||
Object.defineProperty(e, "dataTransfer", { value: { types, dropEffect: "copy" } });
|
||||
document.body.dispatchEvent(e);
|
||||
return e;
|
||||
};
|
||||
expect(missed("drop", ["Files"]).defaultPrevented).toBe(true);
|
||||
expect(missed("dragover", ["Files"]).defaultPrevented).toBe(true);
|
||||
// A drag of something else (a link, text) is the page's own business.
|
||||
expect(missed("drop", ["text/plain"]).defaultPrevented).toBe(false);
|
||||
cleanup();
|
||||
expect(missed("drop", ["Files"]).defaultPrevented).toBe(false);
|
||||
});
|
||||
|
||||
it("a file larger than the import takes is refused before it is sent, naming the limit", async () => {
|
||||
mount(st({ max_bytes: 10, run: run("ok"), watched: {} }));
|
||||
await waitFor(() => expect(line()).toBeTruthy());
|
||||
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
|
||||
const send = vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
|
||||
await drop([file("watch-history.json", 11)]);
|
||||
expect(send).not.toHaveBeenCalled();
|
||||
expect(mocks.notify).toHaveBeenCalledWith({
|
||||
level: "error",
|
||||
message: "settings.takeout.tooLarge(watch-history.json,10 B)",
|
||||
});
|
||||
// Nothing was taken: the zone stays open for the right file, and so after a refusal.
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
send.mockRejectedValue(new HttpError(422, "refused"));
|
||||
await drop([file("bad.html")]);
|
||||
await waitFor(() => expect(mocks.notifyActionError).toHaveBeenCalled());
|
||||
expect(zone()).toBeTruthy();
|
||||
});
|
||||
|
||||
it("the focus follows: New import hands it to the zone, the zone back to New import", async () => {
|
||||
const spy = mount(st({ run: run("ok", { finished_at: "OLD" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("OLD"));
|
||||
fireEvent.click(screen.getByRole("button", { name: "settings.takeout.newImport" }));
|
||||
expect(document.activeElement).toBe(zone());
|
||||
|
||||
vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
|
||||
spy.mockResolvedValue(
|
||||
st({ queued: true, run: run("ok", { finished_at: "OLD" }), watched: {} }),
|
||||
);
|
||||
await drop([file("watch-history.json")]);
|
||||
// While it runs the zone stays, unavailable, and keeps the focus.
|
||||
expect(document.activeElement).toBe(zone());
|
||||
expect(unavailable(zone())).toBe(true);
|
||||
spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 });
|
||||
expect(zone()).toBeNull();
|
||||
expect(document.activeElement).toBe(
|
||||
screen.getByRole("button", { name: "settings.takeout.newImport" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("a focus the user moved elsewhere meanwhile is not taken back", async () => {
|
||||
const spy = mount(st({ run: null, watched: {} }));
|
||||
await waitFor(() => expect(zone()).toBeTruthy());
|
||||
const elsewhere = document.body.appendChild(document.createElement("button"));
|
||||
zone()!.focus();
|
||||
vi.spyOn(api, "takeoutUpload").mockResolvedValue(uploaded(1));
|
||||
spy.mockResolvedValue(st({ queued: true, run: null, watched: {} }));
|
||||
await drop([file("watch-history.json")]);
|
||||
elsewhere.focus();
|
||||
spy.mockResolvedValue(st({ run: run("ok", { finished_at: "NEW" }), watched: {} }));
|
||||
await waitFor(() => expect(line()).toContain("NEW"), { timeout: 3000 });
|
||||
expect(document.activeElement).toBe(elsewhere);
|
||||
elsewhere.remove();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,252 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { RefreshCw, UploadCloud } from "lucide-react";
|
||||
import { Section } from "../../components/ui/form";
|
||||
import { Button } from "../../components/ui/Button";
|
||||
import { DropZone } from "../../components/ui/DropZone";
|
||||
import { StateMessage } from "../../components/ui/QueryState";
|
||||
import { api, type TakeoutStatus, type TakeoutUpload } from "../../lib/api";
|
||||
import type { TAKEOUT_OUTCOMES } from "../../lib/api/feed";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
import { notify } from "../../lib/notifications";
|
||||
import { notifyActionError } from "../../lib/actionErrors";
|
||||
import { formatBytes, formatDateTime, formatNumber } from "../../lib/format";
|
||||
import { importBusy, invalidateImported, watchTakeout } from "./takeoutWatch";
|
||||
|
||||
// Settings › Account: the user's Google Takeout export, imported. What they watched on YouTube and
|
||||
// their YouTube Watch later list reach no API, so the export's two files are uploaded here, one
|
||||
// request each; the server tells them apart and runs the import in the background, and its outcome
|
||||
// also reaches the inbox. The import has no steps to count (the whole history is applied in one
|
||||
// go), so while it runs the section says so and asks again until it is done (`takeoutWatch`).
|
||||
|
||||
// The outcome counts, in the order they are said: the history's, then Watch later's. A count the
|
||||
// import did not produce is not said at all.
|
||||
const HISTORY = [
|
||||
["watched", "watched"],
|
||||
["waiting", "waiting"],
|
||||
["short", "short"],
|
||||
["kept", "kept"],
|
||||
] as const;
|
||||
const WATCH_LATER = [
|
||||
["added", "added"],
|
||||
["present", "present"],
|
||||
["missing", "missing"],
|
||||
["waiting", "laterWaiting"],
|
||||
] as const;
|
||||
|
||||
// Every outcome the server can name is said by one of the two lists (feed.ts holds them to it).
|
||||
type TakeoutOutcome = (typeof TAKEOUT_OUTCOMES)[number];
|
||||
type Said = (typeof HISTORY)[number][0] | (typeof WATCH_LATER)[number][0];
|
||||
const everyOutcomeSaid: [Exclude<TakeoutOutcome, Said>] extends [never] ? true : never = true;
|
||||
void everyOutcomeSaid;
|
||||
|
||||
function countParts(s: TakeoutStatus, say: (key: string, n: number) => string): string[] {
|
||||
const parts: string[] = [];
|
||||
for (const [outcome, key] of HISTORY) {
|
||||
const n = s.watched?.[outcome] ?? 0;
|
||||
if (n) parts.push(say(key, n));
|
||||
}
|
||||
for (const [outcome, key] of WATCH_LATER) {
|
||||
const n = s.watch_later?.[outcome] ?? 0;
|
||||
if (n) parts.push(say(key, n));
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
|
||||
export default function TakeoutImport() {
|
||||
const { t, i18n } = useTranslation();
|
||||
const lang = i18n.language;
|
||||
const qc = useQueryClient();
|
||||
const status = useQuery({ queryKey: qk.takeout(), queryFn: api.takeoutStatus });
|
||||
const [uploading, setUploading] = useState<string | null>(null);
|
||||
// "New import" reopens the drop zone over a finished one; before the first import it is open.
|
||||
const [reopened, setReopened] = useState(false);
|
||||
const run = status.data?.run ?? null;
|
||||
// Queued counts too: until its run row is written, `run` is still the import before it.
|
||||
const busy = importBusy(status.data);
|
||||
const running = uploading !== null || busy;
|
||||
const finishedAt = run?.finished_at ?? null;
|
||||
|
||||
// A run under way is watched to its end, even if the user leaves Settings meanwhile.
|
||||
useEffect(() => {
|
||||
if (busy) void watchTakeout(qc);
|
||||
}, [busy, qc]);
|
||||
// One that ended while nothing here watched it (a reload, another tab) has changed what the feed
|
||||
// and Watch later show too: those lists are read again when next opened, not from the cache.
|
||||
useEffect(() => {
|
||||
if (finishedAt) invalidateImported(qc);
|
||||
}, [finishedAt, qc]);
|
||||
|
||||
// A file dropped beside the zone, or while it is not there, is kept from the browser: it would
|
||||
// open the file in the tab, unloading the app and the upload under way. The zone takes its own.
|
||||
useEffect(() => {
|
||||
const missed = (e: DragEvent) => {
|
||||
if (e.defaultPrevented || !e.dataTransfer?.types.includes("Files")) return;
|
||||
e.preventDefault();
|
||||
if (e.type === "dragover") e.dataTransfer.dropEffect = "none";
|
||||
};
|
||||
window.addEventListener("dragover", missed);
|
||||
window.addEventListener("drop", missed);
|
||||
return () => {
|
||||
window.removeEventListener("dragover", missed);
|
||||
window.removeEventListener("drop", missed);
|
||||
};
|
||||
}, []);
|
||||
|
||||
// Where the focus goes when what holds it leaves: "New import" hands it to the zone it opens, and
|
||||
// the zone, gone once the import it started is done, hands it to "New import".
|
||||
const zone = useRef<HTMLButtonElement | null>(null);
|
||||
const focusNext = useRef<"zone" | "newImport" | null>(null);
|
||||
const focusIfNext = (which: "zone" | "newImport", el: HTMLButtonElement | null) => {
|
||||
if (!el || focusNext.current !== which) return;
|
||||
focusNext.current = null;
|
||||
// Only focus that was lost with the control that held it: never taken from where the user went.
|
||||
if (document.activeElement === null || document.activeElement === document.body) el.focus();
|
||||
};
|
||||
const zoneRef = (el: HTMLButtonElement | null) => {
|
||||
zone.current = el;
|
||||
focusIfNext("zone", el);
|
||||
};
|
||||
const newImportRef = (el: HTMLButtonElement | null) => focusIfNext("newImport", el);
|
||||
|
||||
const upload = async (files: File[]) => {
|
||||
if (document.activeElement === zone.current) focusNext.current = "newImport";
|
||||
let taken = false;
|
||||
for (const file of files) if (await uploadOne(file)) taken = true;
|
||||
// The new run read first, then the upload's line let go: the other way round the previous
|
||||
// import's line showed for a moment, as if this upload had done nothing.
|
||||
await qc.invalidateQueries({ queryKey: qk.takeout() });
|
||||
setUploading(null);
|
||||
// Nothing taken, nothing imported: the zone stays open for the right file.
|
||||
if (taken) setReopened(false);
|
||||
else focusNext.current = null;
|
||||
};
|
||||
/** Send one file; whether the server took it. */
|
||||
const uploadOne = async (file: File): Promise<boolean> => {
|
||||
const max = status.data?.max_bytes ?? Infinity;
|
||||
if (file.size > max) {
|
||||
// Refused here, not after minutes of upload, and with the limit named.
|
||||
notify({
|
||||
level: "error",
|
||||
message: t("settings.takeout.tooLarge", { name: file.name, limit: formatBytes(max, lang) }),
|
||||
});
|
||||
return false;
|
||||
}
|
||||
setUploading(file.name);
|
||||
let res: TakeoutUpload | null = null;
|
||||
try {
|
||||
res = await api.takeoutUpload(file);
|
||||
} catch (e) {
|
||||
// A refusal names its reason (not a Takeout file, nothing in it); the next file still goes.
|
||||
notifyActionError(e, t("errors.writeFailed.takeoutUpload"));
|
||||
}
|
||||
if (res === null) return false;
|
||||
notify({
|
||||
level: "success",
|
||||
message: t("settings.takeout.read", {
|
||||
name: file.name,
|
||||
count: res.videos,
|
||||
n: formatNumber(res.videos, lang),
|
||||
}),
|
||||
});
|
||||
return true;
|
||||
};
|
||||
|
||||
const say = (key: string, n: number) =>
|
||||
t(`settings.takeout.count.${key}`, { count: n, n: formatNumber(n, lang) });
|
||||
const when = (iso: string | null) => (iso ? formatDateTime(iso, lang) : "");
|
||||
// Not while the last import is still being asked for: a zone that flashes before the line that
|
||||
// replaces it would offer a first import to someone who has done one. While an import runs the
|
||||
// zone stays, unavailable: the control that started it keeps the focus.
|
||||
const showZone = status.isSuccess && (running || run === null || reopened);
|
||||
// A failed re-read keeps the previous answer: it is not shown under the line saying the read
|
||||
// failed, as if it were current — a run's spinner no more than its outcome.
|
||||
const current = !status.isError || uploading !== null;
|
||||
|
||||
return (
|
||||
<Section title={t("settings.takeout.title")}>
|
||||
<p className="text-xs text-muted leading-relaxed mb-2">{t("settings.takeout.intro")}</p>
|
||||
|
||||
{status.isError && (
|
||||
<StateMessage compact tone="error" onRetry={() => void status.refetch()}>
|
||||
{t("settings.takeout.loadFailed")}
|
||||
</StateMessage>
|
||||
)}
|
||||
|
||||
{showZone && (
|
||||
<>
|
||||
<DropZone
|
||||
ref={zoneRef}
|
||||
className="px-4 py-5"
|
||||
multiple
|
||||
accept=".json,.csv"
|
||||
disabled={running}
|
||||
onFiles={upload}
|
||||
>
|
||||
{(dragOver) => (
|
||||
<>
|
||||
<UploadCloud className="w-6 h-6 text-muted" />
|
||||
<span className="text-sm text-muted">
|
||||
{dragOver ? t("settings.takeout.dropActive") : t("settings.takeout.drop")}
|
||||
</span>
|
||||
</>
|
||||
)}
|
||||
</DropZone>
|
||||
<details className="mt-2">
|
||||
<summary className="text-xs text-muted cursor-pointer">
|
||||
{t("settings.takeout.howTo")}
|
||||
</summary>
|
||||
<p className="mt-1 text-xs text-muted leading-relaxed">
|
||||
{t("settings.takeout.howToBody")}
|
||||
</p>
|
||||
</details>
|
||||
</>
|
||||
)}
|
||||
|
||||
<div className="mt-2 flex items-center justify-between gap-3">
|
||||
{/* One live region for the run and its outcome, mounted throughout: what replaces
|
||||
"Processing…" is announced, which a region mounted with its text is not. */}
|
||||
<div role="status" className="min-w-0 text-xs text-muted">
|
||||
{running && current ? (
|
||||
<span className="flex items-center gap-2" data-testid="takeout-running">
|
||||
<RefreshCw className="w-3.5 h-3.5 animate-spin" />
|
||||
{uploading !== null
|
||||
? t("settings.takeout.uploading", { name: uploading })
|
||||
: t("settings.takeout.running")}
|
||||
</span>
|
||||
) : (
|
||||
!running &&
|
||||
run &&
|
||||
status.data &&
|
||||
current && (
|
||||
<p data-testid="takeout-last">
|
||||
{run.status === "error"
|
||||
? t("settings.takeout.failed", { when: when(run.finished_at) })
|
||||
: run.status === "interrupted"
|
||||
? t("settings.takeout.interrupted", { when: when(run.started_at) })
|
||||
: [
|
||||
t("settings.takeout.last", { when: when(run.finished_at) }),
|
||||
...countParts(status.data, say),
|
||||
].join(" · ")}
|
||||
</p>
|
||||
)
|
||||
)}
|
||||
</div>
|
||||
{!running && run && status.data && current && !reopened && (
|
||||
<Button
|
||||
ref={newImportRef}
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => {
|
||||
focusNext.current = "zone";
|
||||
setReopened(true);
|
||||
}}
|
||||
>
|
||||
{t("settings.takeout.newImport")}
|
||||
</Button>
|
||||
)}
|
||||
</div>
|
||||
</Section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
// @vitest-environment jsdom
|
||||
//
|
||||
// A Takeout import is watched to its end whether or not its section is still on screen, and what it
|
||||
// changed is read afresh then: the feed and its counts, Watch later, and the playlists' sizes.
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { QueryClient } from "@tanstack/react-query";
|
||||
import { api, type TakeoutStatus } from "../../lib/api";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
import { importBusy, watchTakeout } from "./takeoutWatch";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
const st = (s: Partial<TakeoutStatus>): TakeoutStatus => ({
|
||||
queued: false,
|
||||
max_bytes: 1,
|
||||
run: null,
|
||||
...s,
|
||||
});
|
||||
const running = {
|
||||
status: "running",
|
||||
started_at: "S",
|
||||
finished_at: null,
|
||||
result: null,
|
||||
failure: null,
|
||||
};
|
||||
|
||||
describe("the Takeout import's watch", () => {
|
||||
it("a queued import is busy as much as a running one", () => {
|
||||
expect(importBusy(st({ queued: true }))).toBe(true);
|
||||
expect(importBusy(st({ run: { ...running, status: "running" } as TakeoutStatus["run"] }))).toBe(
|
||||
true,
|
||||
);
|
||||
expect(importBusy(st({ run: { ...running, status: "ok" } as TakeoutStatus["run"] }))).toBe(
|
||||
false,
|
||||
);
|
||||
expect(importBusy(undefined)).toBe(false);
|
||||
});
|
||||
|
||||
it("asks until the import is done, then reads again everything it changed", async () => {
|
||||
vi.useFakeTimers();
|
||||
const qc = new QueryClient();
|
||||
qc.setQueryData(qk.takeout(), st({ queued: true }));
|
||||
const answers = [st({ run: running as TakeoutStatus["run"] }), st({})];
|
||||
const read = vi.spyOn(api, "takeoutStatus").mockImplementation(async () => answers.shift()!);
|
||||
const stale = vi.spyOn(qc, "invalidateQueries");
|
||||
|
||||
const done = watchTakeout(qc);
|
||||
void watchTakeout(qc); // a second call while it watches is the same watch
|
||||
await vi.advanceTimersByTimeAsync(1500);
|
||||
expect(read).toHaveBeenCalledTimes(1);
|
||||
expect(stale).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1500);
|
||||
await done;
|
||||
|
||||
expect(read).toHaveBeenCalledTimes(2);
|
||||
expect(stale.mock.calls.map(([f]) => f?.queryKey)).toEqual([
|
||||
qk.feed(),
|
||||
qk.feedSummary(),
|
||||
qk.feedYears(),
|
||||
qk.playlist(),
|
||||
qk.playlists(),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { QueryClient } from "@tanstack/react-query";
|
||||
import { api, type TakeoutStatus } from "../../lib/api";
|
||||
import { qk } from "../../lib/queryKeys";
|
||||
|
||||
// A Takeout import, watched to its end: its status is asked again until no run is queued or going,
|
||||
// then the lists it changed are read afresh. It lives outside the section on purpose: the user may
|
||||
// leave Settings while the import runs, and the feed they go to must not keep showing as new what
|
||||
// the import has just marked watched.
|
||||
|
||||
const POLL_MS = 1500;
|
||||
|
||||
export const importBusy = (s: TakeoutStatus | undefined): boolean =>
|
||||
!!s && (s.queued || s.run?.status === "running");
|
||||
|
||||
let watching = false;
|
||||
|
||||
const pause = () => new Promise<void>((done) => setTimeout(done, POLL_MS));
|
||||
|
||||
async function read(qc: QueryClient): Promise<TakeoutStatus | undefined> {
|
||||
try {
|
||||
return await qc.fetchQuery({
|
||||
queryKey: qk.takeout(),
|
||||
queryFn: api.takeoutStatus,
|
||||
staleTime: 0,
|
||||
});
|
||||
} catch {
|
||||
// A read that failed is the section's to say (its query holds the error); the watch asks again.
|
||||
return qc.getQueryData<TakeoutStatus>(qk.takeout());
|
||||
}
|
||||
}
|
||||
|
||||
/** Mark stale everything an import may have changed: the feed and its counts, Watch later, and the
|
||||
* playlists that show its size. */
|
||||
export function invalidateImported(qc: QueryClient): void {
|
||||
for (const queryKey of [
|
||||
qk.feed(),
|
||||
qk.feedSummary(),
|
||||
qk.feedYears(),
|
||||
qk.playlist(),
|
||||
qk.playlists(),
|
||||
])
|
||||
void qc.invalidateQueries({ queryKey });
|
||||
}
|
||||
|
||||
/** Watch the user's import until it is done (once: a second call while it watches is that watch),
|
||||
* then mark stale what it changed. */
|
||||
export async function watchTakeout(qc: QueryClient): Promise<void> {
|
||||
if (watching) return;
|
||||
watching = true;
|
||||
let status = qc.getQueryData<TakeoutStatus>(qk.takeout());
|
||||
while (importBusy(status)) {
|
||||
await pause();
|
||||
status = await read(qc);
|
||||
}
|
||||
watching = false;
|
||||
invalidateImported(qc);
|
||||
}
|
||||
+27
-1
@@ -28,7 +28,7 @@ from app.db import SessionLocal
|
||||
import app.tables # noqa: F401 — every table, before the first query
|
||||
from app.catalog.models import Channel, Subscription, Video
|
||||
from app.models import AppState, User
|
||||
from app.reader.models import Playlist, PlaylistItem, VideoState
|
||||
from app.reader.models import Playlist, PlaylistItem, TakeoutItem, VideoState
|
||||
from app.security import hash_password
|
||||
|
||||
EMAIL = os.environ.get("E2E_EMAIL", "e2e@siftlode.local")
|
||||
@@ -118,6 +118,9 @@ def _seed_account(db, email: str, role: str, display_name: str) -> dict:
|
||||
# - one WATCHED video from the second channel, so the "watched" show-filter is a
|
||||
# real, non-empty narrowing of the feed (and the "all" vs "unwatched" split exists).
|
||||
db.execute(delete(VideoState).where(VideoState.user_id == user.id))
|
||||
# A Takeout import's kept videos too: a settled row keeps its outcome over a re-upload, so the
|
||||
# import spec's counts would otherwise be the previous run's, not this one's.
|
||||
db.execute(delete(TakeoutItem).where(TakeoutItem.user_id == user.id))
|
||||
|
||||
def _newest(channel_id: str, exclude: set[str]) -> Video | None:
|
||||
conds = [
|
||||
@@ -178,6 +181,28 @@ def _seed_account(db, email: str, role: str, display_name: str) -> dict:
|
||||
saved = svid.id
|
||||
db.commit()
|
||||
|
||||
# 5. Videos the Takeout import marks watched: from the subscribed channels, and already cleared by
|
||||
# the Shorts probe — one not yet probed would wait for it, and the import spec would count
|
||||
# nothing. One per browser project: they share this account, and a video imported once is not
|
||||
# marked again.
|
||||
known = (
|
||||
list(
|
||||
db.scalars(
|
||||
select(Video.id)
|
||||
.where(
|
||||
Video.channel_id.in_(channel_ids),
|
||||
Video.shorts_probed.is_(True),
|
||||
Video.is_short.is_(False),
|
||||
Video.id.not_in(used | {saved} if saved else used),
|
||||
)
|
||||
.order_by(Video.published_at.desc())
|
||||
.limit(5)
|
||||
)
|
||||
)
|
||||
if channel_ids
|
||||
else []
|
||||
)
|
||||
|
||||
return {
|
||||
"user_id": user.id,
|
||||
"email": email,
|
||||
@@ -185,6 +210,7 @@ def _seed_account(db, email: str, role: str, display_name: str) -> dict:
|
||||
"in_progress": in_progress,
|
||||
"watched": watched,
|
||||
"saved": saved,
|
||||
"takeout_known": known,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user