peter
9f580fd8c8
- Uploads: one at a time per user and ten an hour, each refused only after its body arrived; the history read an entry at a time (32 MiB cap), a Watch later list capped at YouTube's 5000, a user's kept videos at 200k; any unreadable shape is a 422, not a 500. - Quota: Watch later lookups stop at the backfill reserve and at a per-user daily share; what is not asked about waits, known videos still go in, and the shorts job retries the rest. - Staging and settling share one per-user advisory lock (no deadlock); a run settles only the rows it read, so a row staged meanwhile is never called gone. - A failed run's notice names a translated failure, never the upstream's text. - The status says when an import is queued; the page watches it to its end even when left, then reads the feed, its counts and the playlists again. - Settings: one live region, the zone kept (unavailable) while a run goes, focus handed back, a missed drop kept from the browser, an oversize file refused before it is sent. - Scheduler and run lists show Takeout runs by the importing user.
411 lines
15 KiB
Python
411 lines
15 KiB
Python
import threading
|
|
from collections.abc import AsyncIterator
|
|
from datetime import date
|
|
from functools import partial
|
|
|
|
from fastapi import APIRouter, Depends, Query, Request
|
|
from pydantic import BaseModel
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.orm.exc import StaleDataError
|
|
|
|
from app import jobruns, quota, scheduler
|
|
from app.auth import current_user
|
|
from app.db import get_db, release_read_connection
|
|
from app.mailtext import lang_for_user
|
|
from app.ratelimit import RateLimiter
|
|
from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years
|
|
from app.models import User
|
|
from app.reader import queries, service, takeout
|
|
from app.routes._common import (
|
|
ProgressIn,
|
|
capped_body,
|
|
drain_body,
|
|
kept_position,
|
|
not_found,
|
|
youtube_refusal,
|
|
)
|
|
from app.sync.videos import parse_iso8601_duration
|
|
from app.utils import iso
|
|
from app.youtube.client import YouTubeClient, YouTubeError
|
|
from app.refusal import refused
|
|
|
|
router = APIRouter(prefix="/api", tags=["feed"])
|
|
|
|
# "saved" used to be a status; it's now membership in the built-in Watch later playlist.
|
|
VALID_STATES = {"new", "watched", "hidden"}
|
|
|
|
|
|
# Shared query parameters for /feed and /feed/count.
|
|
def _feed_params(
|
|
tags: list[int] = Query(default=[]),
|
|
# Per-category tag match: each of language / topic / other independently ORs (any) or ANDs (all)
|
|
# its own selected tags. Defaults to "or". (Superseded the single global `tag_mode`.)
|
|
tag_mode_language: str = "or",
|
|
tag_mode_topic: str = "or",
|
|
tag_mode_other: str = "or",
|
|
# "No tag" filter: keep only channels the user has not personally tagged (no user-owned ChannelTag).
|
|
untagged: bool = False,
|
|
channel_ids: list[str] = Query(default=[]),
|
|
q: str | None = None,
|
|
min_duration: int | None = None,
|
|
max_duration: int | None = None,
|
|
max_age_days: int | None = None,
|
|
published_after: date | None = None,
|
|
published_before: date | None = None,
|
|
show_normal: bool = True,
|
|
include_shorts: bool = False,
|
|
include_live: bool = False,
|
|
show: str = "unwatched",
|
|
scope: str = "my",
|
|
library_source: str = "organic",
|
|
) -> dict:
|
|
return {
|
|
"tags": tags,
|
|
"tag_modes": {
|
|
"language": tag_mode_language,
|
|
"topic": tag_mode_topic,
|
|
"other": tag_mode_other,
|
|
},
|
|
"untagged": untagged,
|
|
"channel_ids": channel_ids,
|
|
"q": q,
|
|
"min_duration": min_duration,
|
|
"max_duration": max_duration,
|
|
"max_age_days": max_age_days,
|
|
"published_after": published_after,
|
|
"published_before": published_before,
|
|
"show_normal": show_normal,
|
|
"include_shorts": include_shorts,
|
|
"include_live": include_live,
|
|
"show": show,
|
|
"scope": scope,
|
|
"library_source": library_source,
|
|
}
|
|
|
|
|
|
@router.get("/feed")
|
|
def get_feed(
|
|
params: dict = Depends(_feed_params),
|
|
sort: str = "newest",
|
|
seed: int = 0,
|
|
limit: int = Query(default=60, le=200),
|
|
cursor: str | None = None,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
try:
|
|
return feed_page(
|
|
db,
|
|
user,
|
|
params,
|
|
sort=sort,
|
|
seed=seed,
|
|
limit=limit,
|
|
cursor=cursor,
|
|
lang=lang_for_user(user),
|
|
)
|
|
except BadCursor:
|
|
# `from None`: a malformed cursor is an expected bad-input signal, not an error to chain.
|
|
raise refused(400, "invalid_cursor") from None
|
|
|
|
|
|
@router.get("/feed/count")
|
|
def get_feed_count(
|
|
params: dict = Depends(_feed_params),
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
# SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference
|
|
# oracle that test_feed_summary_matches_count_plus_facets asserts /feed/summary's `count` against;
|
|
# remove both together with that test if this endpoint is ever retired.
|
|
return {"count": feed_count(db, user, **params)}
|
|
|
|
|
|
@router.get("/facets")
|
|
def get_facets(
|
|
params: dict = Depends(_feed_params),
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Per-tag channel counts for the *current* filter context, so the sidebar can show
|
|
live counts and drop chips that no longer match anything. Each category is counted with
|
|
every other filter applied (scope, channel, date, content type, search, watch state,
|
|
and the other category's tags) but its own selections ignored — standard drill-down
|
|
faceting, so selecting one topic doesn't zero out the rest of the topics.
|
|
|
|
The count is the number of distinct channels that have at least one video in the current
|
|
view, matching the existing channel-count chip semantics. JSON object keys are strings
|
|
(tag ids).
|
|
|
|
SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference
|
|
oracle for test_feed_summary_matches_count_plus_facets (it asserts /feed/summary's `facets`
|
|
against these `counts`); retire the two together."""
|
|
return {"counts": facet_counts(db, user, **params)}
|
|
|
|
|
|
@router.get("/feed/summary")
|
|
def get_feed_summary(
|
|
params: dict = Depends(_feed_params),
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""The sidebar's total video count + per-tag channel facet counts in ONE request and ONE grouped
|
|
query, replacing the separate /feed/count (1 pass) + /facets (3 passes) the FE fired on every filter
|
|
change (PERF-13). Response: {"count": <videos>, "facets": {<tag_id>: <channel_count>}} — the same
|
|
numbers /feed/count's `count` and /facets' `counts` return, so the two are interchangeable."""
|
|
return feed_summary(db, user, **params)
|
|
|
|
|
|
@router.get("/feed/years")
|
|
def get_feed_years(
|
|
params: dict = Depends(_feed_params),
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Per-year video counts for the year control: the quick recent-year chips and the
|
|
"more…" per-year-counts dialog. Every filter EXCEPT the publish-date narrowing applies, so picking a
|
|
year re-queries the feed but not this list. Response: {"years": [{"year": <int>, "count": <int>}]},
|
|
newest year first."""
|
|
return {"years": feed_years(db, user, **params)}
|
|
|
|
|
|
def _video_or_404(db: Session, video_id: str) -> None:
|
|
if queries.video(db, video_id) is None:
|
|
raise not_found("video")
|
|
|
|
|
|
class VideoStateIn(BaseModel):
|
|
status: str
|
|
|
|
|
|
@router.post("/videos/{video_id}/state")
|
|
def set_video_state(
|
|
video_id: str,
|
|
body: VideoStateIn,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
status = body.status
|
|
if status not in VALID_STATES:
|
|
raise refused(400, "invalid_status")
|
|
_video_or_404(db, video_id)
|
|
if status == "new":
|
|
try:
|
|
service.unmark(db, user, video_id)
|
|
db.commit()
|
|
# silent-ok: the documented lost race — a concurrent player checkpoint already removed or
|
|
# moved this row (`service.unmark`). The state the caller asked for holds either way.
|
|
except StaleDataError:
|
|
db.rollback()
|
|
return {"ok": True, "video_id": video_id, "status": "new"}
|
|
service.mark(db, user, video_id, status)
|
|
db.commit()
|
|
return {"ok": True, "video_id": video_id, "status": status}
|
|
|
|
|
|
@router.delete("/videos/{video_id}/state")
|
|
def clear_video_state(
|
|
video_id: str,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Reset a video to pristine for this user — drop the whole VideoState row (status,
|
|
watch position and watched_at), as if it had never been opened. Saved/playlist membership
|
|
lives elsewhere and is untouched."""
|
|
try:
|
|
service.clear_state(db, user, video_id)
|
|
db.commit()
|
|
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
|
|
except StaleDataError:
|
|
db.rollback()
|
|
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
|
|
|
|
|
@router.post("/videos/{video_id}/progress")
|
|
def set_video_progress(
|
|
video_id: str,
|
|
body: ProgressIn,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Checkpoint the in-app player's resume position (called periodically while playing).
|
|
Stores a per-user position on the video_states row without touching watch status, so a
|
|
partially-watched video can render a progress bar and match the "in progress" filter.
|
|
Trivially-early and near-finished positions clear the position rather than store it."""
|
|
position = kept_position(body)
|
|
_video_or_404(db, video_id)
|
|
if not position:
|
|
try:
|
|
service.set_progress(db, user, video_id, 0)
|
|
db.commit()
|
|
# silent-ok: a concurrent state change (the end-of-playback "watched" mark) removed or
|
|
# updated this row already — the lost race; the response stays true.
|
|
except StaleDataError:
|
|
db.rollback()
|
|
return {"ok": True, "video_id": video_id, "position_seconds": 0}
|
|
service.set_progress(db, user, video_id, position)
|
|
db.commit()
|
|
return {"ok": True, "video_id": video_id, "position_seconds": position}
|
|
|
|
|
|
@router.get("/videos/{video_id}")
|
|
def get_video_detail(
|
|
video_id: str,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""On-demand detail (description, like count) — kept out of the feed list payload
|
|
so the feed stays lean; fetched lazily, e.g. for the title hover popover.
|
|
|
|
Videos we already store are served from the DB for free. A video that isn't in
|
|
our DB (e.g. a YouTube link inside another video's description that the in-app
|
|
player navigated to) is resolved via the YouTube API (videos.list, 1 unit,
|
|
attributed to the requesting user)."""
|
|
v = queries.video(db, video_id)
|
|
if v is not None:
|
|
return {
|
|
"id": v.id,
|
|
"description": v.description,
|
|
"like_count": v.like_count,
|
|
"in_db": True,
|
|
"channel_id": v.channel_id,
|
|
"channel_title": v.channel.title if v.channel else None,
|
|
"published_at": iso(v.published_at),
|
|
"view_count": v.view_count,
|
|
"duration_seconds": v.duration_seconds,
|
|
}
|
|
|
|
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit.
|
|
try:
|
|
with (
|
|
quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP),
|
|
YouTubeClient(db, user) as yt,
|
|
):
|
|
items = yt.get_videos([video_id])
|
|
except YouTubeError as exc:
|
|
raise youtube_refusal(exc) from exc
|
|
if not items:
|
|
raise not_found("video")
|
|
|
|
snippet = items[0].get("snippet", {})
|
|
stats = items[0].get("statistics", {})
|
|
likes = stats.get("likeCount")
|
|
views = stats.get("viewCount")
|
|
return {
|
|
"id": video_id,
|
|
"description": snippet.get("description"),
|
|
"like_count": int(likes) if likes is not None else None,
|
|
"in_db": False,
|
|
"channel_id": snippet.get("channelId"),
|
|
"channel_title": snippet.get("channelTitle"),
|
|
"published_at": snippet.get("publishedAt"),
|
|
"view_count": int(views) if views is not None else None,
|
|
"duration_seconds": parse_iso8601_duration(
|
|
items[0].get("contentDetails", {}).get("duration")
|
|
),
|
|
}
|
|
|
|
|
|
# --- Google Takeout ------------------------------------------------------------------------------
|
|
|
|
|
|
# One upload at a time per user, and ten an hour: a file is held whole while it is read, so these
|
|
# bound what one account can make the process hold. The section uploads its files one after another.
|
|
_takeout_limiter = RateLimiter(max_events=10, window_seconds=3600)
|
|
_takeout_uploading: set[int] = set()
|
|
_takeout_uploading_guard = threading.Lock()
|
|
|
|
|
|
async def _takeout_body(
|
|
request: Request, user: User = Depends(current_user), db: Session = Depends(get_db)
|
|
) -> AsyncIterator[bytes]:
|
|
# The sign-in check has run, and it was a SELECT: hand the connection back before a file of
|
|
# megabytes arrives, so a slow upload does not hold one `idle in transaction`.
|
|
release_read_connection(db)
|
|
with _takeout_uploading_guard:
|
|
busy = user.id in _takeout_uploading
|
|
if not busy:
|
|
_takeout_uploading.add(user.id)
|
|
if busy or not _takeout_limiter.allow(str(user.id)):
|
|
if not busy:
|
|
with _takeout_uploading_guard:
|
|
_takeout_uploading.discard(user.id)
|
|
# Read to the end before answering: refused mid-upload, the browser reports a network
|
|
# failure instead of the refusal (capped_stream_to says why).
|
|
await drain_body(request)
|
|
if busy:
|
|
raise refused(429, "takeout_busy")
|
|
raise refused(429, "takeout_cooldown")
|
|
try:
|
|
yield await capped_body(request, takeout.MAX_BYTES)
|
|
finally:
|
|
with _takeout_uploading_guard:
|
|
_takeout_uploading.discard(user.id)
|
|
|
|
|
|
@router.post("/takeout")
|
|
def upload_takeout(
|
|
# The sign-in check FIRST: the body's own dependency needs the signed-in user, so a stranger's
|
|
# file is refused before a byte of it is read.
|
|
user: User = Depends(current_user),
|
|
body: bytes = Depends(_takeout_body),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""One file of the user's Google Takeout export — the watch history or the Watch later list.
|
|
Its videos are kept and the import runs in the background; its outcome reaches the user's inbox."""
|
|
try:
|
|
parsed = takeout.parse(body)
|
|
except takeout.TakeoutError as exc:
|
|
if exc.reason == "no_videos":
|
|
raise refused(422, "takeout_no_videos") from None
|
|
if exc.reason == "too_many":
|
|
raise refused(422, "takeout_list_too_long", limit=takeout.WATCH_LATER_MAX) from None
|
|
raise refused(422, "takeout_unreadable") from None
|
|
try:
|
|
takeout.stage(db, user, parsed)
|
|
except takeout.TooMany:
|
|
db.rollback()
|
|
raise refused(422, "takeout_too_many", limit=takeout.ROWS_MAX) from None
|
|
db.commit()
|
|
scheduler.run_ad_hoc(
|
|
scheduler.TAKEOUT_IMPORT,
|
|
partial(takeout.run_import, user.id),
|
|
actor_id=user.id,
|
|
target=takeout.run_target(user.id),
|
|
notify=True,
|
|
)
|
|
return {
|
|
"ok": True,
|
|
"kind": parsed.kind,
|
|
"videos": len(parsed.videos),
|
|
"entries": parsed.entries,
|
|
"ads": parsed.ads,
|
|
"other": parsed.other,
|
|
"malformed": parsed.malformed,
|
|
}
|
|
|
|
|
|
@router.get("/takeout")
|
|
def takeout_status(user: User = Depends(current_user), db: Session = Depends(get_db)):
|
|
"""What the user's Takeout import has done: its last run (none before the first upload), and
|
|
every video kept from the export, by kind and outcome — a video the catalog did not know yet
|
|
counts as "waiting" until it does, and is marked then."""
|
|
target = takeout.run_target(user.id)
|
|
run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, target)
|
|
return {
|
|
# Queued and not started: its run row is not written yet, and the last run is still the
|
|
# previous one, so the page waits for it rather than taking that one for this upload's.
|
|
"queued": scheduler.ad_hoc_queued(scheduler.TAKEOUT_IMPORT, target),
|
|
"max_bytes": takeout.MAX_BYTES,
|
|
"run": None
|
|
if run is None
|
|
else {
|
|
"status": run.status,
|
|
"started_at": iso(run.started_at),
|
|
"finished_at": iso(run.finished_at),
|
|
"result": run.result,
|
|
"failure": run.failure,
|
|
},
|
|
**queries.takeout_summary(db, user),
|
|
}
|