peter
9c5f81fd94
- An ad-hoc request made while its target's run is under way is the run after it: the scheduler dropped a second fire of a running id, so a second Takeout file was never imported and its key stayed queued until a restart. - Un-marking or resetting a video is remembered (video_unmarks): an import of an older YouTube view no longer marks it watched again; un-marks take the user's Takeout lock. - One user's failure in the Shorts job's Takeout pass is theirs: the job keeps its own result, the others are settled, and a failed lookup spends none of the daily share. - The upload reads the user's id before giving the read connection back. - A watched row waits on a discovery video (search or explore) until the catalog keeps it, so a mark never pins it against the cleanup jobs. - The history is decoded a window at a time, a Watch later file over 2 MiB is refused unread, and two uploads at once across all users (takeout_server_busy). - A Watch later list goes in as one presence read and one insert.
280 lines
14 KiB
Python
280 lines
14 KiB
Python
"""Reader's tables: what a user does with the shared catalog — where they are in a video
|
|
(`VideoState`), their playlists, their saved views, their labels on channels (`Tag`, `ChannelTag`)
|
|
and the comfort shelf on Home. Only this unit writes them; another unit reads them along a declared
|
|
edge, or asks this unit's service."""
|
|
|
|
from datetime import datetime
|
|
from enum import StrEnum
|
|
|
|
from sqlalchemy import (
|
|
Boolean,
|
|
CheckConstraint,
|
|
DateTime,
|
|
Float,
|
|
ForeignKey,
|
|
Index,
|
|
Integer,
|
|
JSON,
|
|
String,
|
|
Text,
|
|
UniqueConstraint,
|
|
func,
|
|
text,
|
|
)
|
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
|
|
|
from app.db import Base
|
|
from app.models import TimestampMixin, UpdatedAtMixin
|
|
|
|
|
|
class Tag(Base, TimestampMixin):
|
|
"""A label. System tags (user_id NULL) are computed automatically and shared by all
|
|
users; user tags (user_id set) are personal. category groups them in the UI."""
|
|
|
|
__tablename__ = "tags"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("users.id", ondelete="CASCADE"), index=True
|
|
)
|
|
name: Mapped[str] = mapped_column(String(64))
|
|
color: Mapped[str | None] = mapped_column(String(16))
|
|
# language | topic | other
|
|
category: Mapped[str] = mapped_column(String(16), default="other", server_default="other")
|
|
|
|
|
|
class ChannelTag(Base):
|
|
"""Links a tag to a channel. source='auto' rows are managed by the auto-tagger and
|
|
regenerated freely; source='user' rows are never touched by the system."""
|
|
|
|
__tablename__ = "channel_tags"
|
|
__table_args__ = (UniqueConstraint("channel_id", "tag_id", "user_id", name="uq_channel_tag"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
channel_id: Mapped[str] = mapped_column(
|
|
ForeignKey("channels.id", ondelete="CASCADE"), index=True
|
|
)
|
|
tag_id: Mapped[int] = mapped_column(ForeignKey("tags.id", ondelete="CASCADE"), index=True)
|
|
# NULL = system/auto link visible to everyone; set = a specific user's link.
|
|
user_id: Mapped[int | None] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
source: Mapped[str] = mapped_column(String(8), default="auto", server_default="auto")
|
|
confidence: Mapped[float | None] = mapped_column(Float)
|
|
|
|
|
|
class VideoState(Base, UpdatedAtMixin):
|
|
"""Per-user watch state for a video. Rows exist only for non-default states; the
|
|
absence of a row means 'new/unseen'."""
|
|
|
|
__tablename__ = "video_states"
|
|
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_user_video"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
|
# new | watched | hidden ("saved" is now Watch-later playlist membership, not a status)
|
|
status: Mapped[str] = mapped_column(String(12), default="new", server_default="new")
|
|
watched_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
# Resume position (seconds into the video) for the in-app player. Orthogonal to
|
|
# status: a partially-watched video keeps status "new" but a non-zero position, so
|
|
# it can render a progress bar and match the "in progress" feed filter. 0 = not
|
|
# started / finished (trivially-early and near-finished positions are not stored).
|
|
# int4: a checkpoint body must stay under POSITION_MAX_S, or the flush is a DataError (a 500).
|
|
position_seconds: Mapped[int] = mapped_column(
|
|
Integer, default=0, server_default="0", nullable=False
|
|
)
|
|
progress_updated_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
|
|
|
|
class VideoUnmark(Base):
|
|
"""When the user last took a video back to "new" here — un-marked it, or reset it. That leaves
|
|
no `VideoState` (its absence is "new"), so without this row a later import of what they watched
|
|
on YouTube could not tell a video they un-marked from one never touched, and marked it watched
|
|
again. Kept by user and video; the video's own deletion takes it."""
|
|
|
|
__tablename__ = "video_unmarks"
|
|
__table_args__ = (UniqueConstraint("user_id", "video_id", name="uq_video_unmark"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
|
at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
|
|
|
|
class Playlist(Base, TimestampMixin, UpdatedAtMixin):
|
|
"""A per-user named, ordered collection of videos from the shared catalog.
|
|
|
|
`kind`: 'user' (normal) or 'watch_later' (built-in, undeletable, one per user).
|
|
`source`: 'local' (created here) or 'youtube' (mirrored from the user's YouTube
|
|
playlists). `yt_playlist_id` links a mirrored/exported playlist to its YouTube id.
|
|
`dirty` marks a YouTube-linked playlist with local edits not yet pushed, so the read
|
|
sync won't clobber them. (source/yt_playlist_id/dirty are forward-looking for the YT
|
|
sync phases; the foundation phase only uses local playlists.)"""
|
|
|
|
__tablename__ = "playlists"
|
|
__table_args__ = (
|
|
# One Watch later per user, held by the database: two first saves at once each made one.
|
|
Index(
|
|
"uq_playlist_watch_later",
|
|
"user_id",
|
|
unique=True,
|
|
postgresql_where=text("kind = 'watch_later'"),
|
|
),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
|
name: Mapped[str] = mapped_column(String(255))
|
|
kind: Mapped[str] = mapped_column(String(16), default="user", server_default="user")
|
|
source: Mapped[str] = mapped_column(String(16), default="local", server_default="local")
|
|
yt_playlist_id: Mapped[str | None] = mapped_column(String(64))
|
|
# True when the current items/name differ from `synced_fingerprint` (unpushed edits).
|
|
dirty: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
# SHA-256 of name + ordered video ids as last synced from / pushed to YouTube; the
|
|
# baseline `dirty` is derived against. NULL = no baseline yet (treated as dirty).
|
|
synced_fingerprint: Mapped[str | None] = mapped_column(String)
|
|
# JSON list of the ordered video ids as of the last sync — the 3-way-merge baseline the
|
|
# two-way YouTube sync classifies each item against (local-add vs YT-add vs real delete).
|
|
# NULL = no baseline yet; the first merge then does a lossless union (never deletes).
|
|
synced_video_ids: Mapped[str | None] = mapped_column(Text)
|
|
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
|
|
items: Mapped[list["PlaylistItem"]] = relationship(
|
|
back_populates="playlist",
|
|
cascade="all, delete-orphan",
|
|
order_by="PlaylistItem.position",
|
|
)
|
|
|
|
|
|
class PlaylistItem(Base):
|
|
"""A video's membership in a playlist, with its order within that playlist."""
|
|
|
|
__tablename__ = "playlist_items"
|
|
__table_args__ = (UniqueConstraint("playlist_id", "video_id", name="uq_playlist_video"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
playlist_id: Mapped[int] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
|
|
video_id: Mapped[str] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"), index=True)
|
|
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
added_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
|
|
|
playlist: Mapped["Playlist"] = relationship(back_populates="items")
|
|
|
|
|
|
class SavedView(Base, TimestampMixin, UpdatedAtMixin):
|
|
"""A per-user named snapshot of a module's filter/sort state (a "smart view").
|
|
|
|
`source` names the module the view belongs to ("feed" | "plex"); `filters` is that module's
|
|
serialized filter JSON the frontend applies with one click. `position` orders them in the sidebar
|
|
(drag-reorder, per source); `is_default` marks the one view applied automatically on load — at most
|
|
one per (user, source), enforced by the API on set. The name is unique per (user, source)."""
|
|
|
|
__tablename__ = "saved_views"
|
|
__table_args__ = (
|
|
UniqueConstraint("user_id", "source", "name", name="uq_user_source_saved_view"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
source: Mapped[str] = mapped_column(String(16), default="feed", server_default="feed")
|
|
name: Mapped[str] = mapped_column(String(80))
|
|
filters: Mapped[dict] = mapped_column(JSON)
|
|
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0")
|
|
is_default: Mapped[bool] = mapped_column(Boolean, default=False, server_default="false")
|
|
|
|
|
|
class ComfortPin(Base, TimestampMixin):
|
|
"""One entry on a user's comfort shelf — the anti-paralysis fallback: "just put on X".
|
|
|
|
Deliberately NOT a boolean on `VideoState`/`PlexState`, and NOT a polymorphic `(kind, ref)` string
|
|
pair. A flag cannot ORDER a shelf and cannot hold a channel or a show — and comfort media is
|
|
usually "put on X", a series or a channel, not one particular video. A string ref would order and
|
|
hold anything, but it would also lose the foreign key: a Plex item dropped from the library, or an
|
|
unsubscribed channel, would leave a dead tile on the one surface whose whole job is to be
|
|
reliable. So instead: one nullable CASCADE-ing FK per pinnable kind, and a check constraint that
|
|
exactly one of them is set.
|
|
|
|
That constraint is the house rule applied to a five-way union — the invariant lives in the schema,
|
|
not in a comment every insert site has to remember. `kind`/`ref` below derive FROM the columns, so
|
|
there is no second vocabulary to keep in sync.
|
|
|
|
NULLs are DISTINCT in a Postgres unique index, so the five per-kind unique constraints each stop a
|
|
double-pin of the same thing without colliding across kinds."""
|
|
|
|
__tablename__ = "comfort_pins"
|
|
__table_args__ = (
|
|
CheckConstraint(
|
|
"num_nonnulls(video_id, channel_id, plex_item_id, plex_show_id, playlist_id) = 1",
|
|
name="ck_comfort_pin_one_ref",
|
|
),
|
|
UniqueConstraint("user_id", "video_id", name="uq_comfort_pin_video"),
|
|
UniqueConstraint("user_id", "channel_id", name="uq_comfort_pin_channel"),
|
|
UniqueConstraint("user_id", "plex_item_id", name="uq_comfort_pin_plex_item"),
|
|
UniqueConstraint("user_id", "plex_show_id", name="uq_comfort_pin_plex_show"),
|
|
UniqueConstraint("user_id", "playlist_id", name="uq_comfort_pin_playlist"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), index=True)
|
|
|
|
video_id: Mapped[str | None] = mapped_column(ForeignKey("videos.id", ondelete="CASCADE"))
|
|
channel_id: Mapped[str | None] = mapped_column(ForeignKey("channels.id", ondelete="CASCADE"))
|
|
plex_item_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("plex_items.id", ondelete="CASCADE")
|
|
)
|
|
plex_show_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("plex_shows.id", ondelete="CASCADE")
|
|
)
|
|
playlist_id: Mapped[int | None] = mapped_column(ForeignKey("playlists.id", ondelete="CASCADE"))
|
|
|
|
# 0-based shelf order. The shelf is short and hand-curated, so a dense integer the reorder
|
|
# endpoint rewrites wholesale beats a fractional index nobody would ever exercise here.
|
|
position: Mapped[int] = mapped_column(Integer, default=0, server_default="0", nullable=False)
|
|
|
|
|
|
class TakeoutKind(StrEnum):
|
|
"""What a Google Takeout row says the user did with a video."""
|
|
|
|
WATCHED = "watched" # history/watch-history.json
|
|
WATCH_LATER = "watch_later" # playlists/Watch later-videos.csv
|
|
|
|
|
|
class TakeoutOutcome(StrEnum):
|
|
"""What the import did with a row; NULL = still waiting for its video to be known (and, for a
|
|
watched one, probed for Shorts)."""
|
|
|
|
WATCHED = "watched" # the video is marked watched at the Takeout time
|
|
KEPT = "kept" # the state the user set here (watched, hidden, or changed later) stays
|
|
SHORT = "short" # a Short: swiping one past is not watching it
|
|
ADDED = "added" # added to Watch later
|
|
PRESENT = "present" # already in Watch later
|
|
MISSING = "missing" # YouTube no longer has the video
|
|
|
|
|
|
class TakeoutItem(Base, TimestampMixin):
|
|
"""One video from a user's Google Takeout export: watched (with when) or in their YouTube Watch
|
|
later list (with when it was added). The rows are both the import job's input and its memory: a
|
|
video the catalog does not know yet waits here, with no outcome, until it enters the catalog.
|
|
The id is not a foreign key for that reason. The channel is kept for a later "channels you
|
|
watch but do not follow" list."""
|
|
|
|
__tablename__ = "takeout_items"
|
|
__table_args__ = (
|
|
UniqueConstraint("user_id", "kind", "video_id", name="uq_takeout_item"),
|
|
CheckConstraint("kind IN ('watched', 'watch_later')", name="ck_takeout_item_kind"),
|
|
# The waiting rows, by video: what a newly known video looks up.
|
|
Index(
|
|
"ix_takeout_items_waiting",
|
|
"video_id",
|
|
postgresql_where=text("outcome IS NULL"),
|
|
),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"))
|
|
kind: Mapped[str] = mapped_column(String(12))
|
|
video_id: Mapped[str] = mapped_column(String(16))
|
|
channel_id: Mapped[str | None] = mapped_column(String(32))
|
|
# Watched: the last time in the export; Watch later: when it was added.
|
|
at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
outcome: Mapped[str | None] = mapped_column(String(12))
|