import threading from collections.abc import AsyncIterator from datetime import date from functools import partial from fastapi import APIRouter, Depends, Path, Query, Request from pydantic import BaseModel from sqlalchemy.orm import Session from sqlalchemy.orm.exc import StaleDataError from app import jobruns, quota, scheduler from app.catalog import queries as catalog_queries, service as catalog_service from app.auth import current_user from app.db import get_db, release_read_connection from app.mailtext import lang_for_user from app.ratelimit import RateLimiter from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years from app.models import User from app.reader import queries, service, takeout from app.routes._common import ( ProgressIn, capped_body, drain_body, kept_position, not_found, youtube_refusal, ) from app.sync.videos import ensure_videos, parse_iso8601_duration from app.utils import iso from app.youtube.client import YouTubeClient, YouTubeError from app.refusal import refused router = APIRouter(prefix="/api", tags=["feed"]) # "saved" used to be a status; it's now membership in the built-in Watch later playlist. VALID_STATES = {"new", "watched", "hidden"} # Shared query parameters for /feed and /feed/count. def _feed_params( tags: list[int] = Query(default=[]), # Per-category tag match: each of language / topic / other independently ORs (any) or ANDs (all) # its own selected tags. Defaults to "or". (Superseded the single global `tag_mode`.) tag_mode_language: str = "or", tag_mode_topic: str = "or", tag_mode_other: str = "or", # "No tag" filter: keep only channels the user has not personally tagged (no user-owned ChannelTag). untagged: bool = False, channel_ids: list[str] = Query(default=[]), q: str | None = None, min_duration: int | None = None, max_duration: int | None = None, max_age_days: int | None = None, published_after: date | None = None, published_before: date | None = None, show_normal: bool = True, include_shorts: bool = False, include_live: bool = False, show: str = "unwatched", scope: str = "my", library_source: str = "organic", ) -> dict: return { "tags": tags, "tag_modes": { "language": tag_mode_language, "topic": tag_mode_topic, "other": tag_mode_other, }, "untagged": untagged, "channel_ids": channel_ids, "q": q, "min_duration": min_duration, "max_duration": max_duration, "max_age_days": max_age_days, "published_after": published_after, "published_before": published_before, "show_normal": show_normal, "include_shorts": include_shorts, "include_live": include_live, "show": show, "scope": scope, "library_source": library_source, } @router.get("/feed") def get_feed( params: dict = Depends(_feed_params), sort: str = "newest", seed: int = 0, limit: int = Query(default=60, le=200), cursor: str | None = None, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: try: return feed_page( db, user, params, sort=sort, seed=seed, limit=limit, cursor=cursor, lang=lang_for_user(user), ) except BadCursor: # `from None`: a malformed cursor is an expected bad-input signal, not an error to chain. raise refused(400, "invalid_cursor") from None @router.get("/feed/count") def get_feed_count( params: dict = Depends(_feed_params), user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: # SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference # oracle that test_feed_summary_matches_count_plus_facets asserts /feed/summary's `count` against; # remove both together with that test if this endpoint is ever retired. return {"count": feed_count(db, user, **params)} @router.get("/facets") def get_facets( params: dict = Depends(_feed_params), user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: """Per-tag channel counts for the *current* filter context, so the sidebar can show live counts and drop chips that no longer match anything. Each category is counted with every other filter applied (scope, channel, date, content type, search, watch state, and the other category's tags) but its own selections ignored — standard drill-down faceting, so selecting one topic doesn't zero out the rest of the topics. The count is the number of distinct channels that have at least one video in the current view, matching the existing channel-count chip semantics. JSON object keys are strings (tag ids). SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference oracle for test_feed_summary_matches_count_plus_facets (it asserts /feed/summary's `facets` against these `counts`); retire the two together.""" return {"counts": facet_counts(db, user, **params)} @router.get("/feed/summary") def get_feed_summary( params: dict = Depends(_feed_params), user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: """The sidebar's total video count + per-tag channel facet counts in ONE request and ONE grouped query, replacing the separate /feed/count (1 pass) + /facets (3 passes) the FE fired on every filter change (PERF-13). Response: {"count": , "facets": {: }} — the same numbers /feed/count's `count` and /facets' `counts` return, so the two are interchangeable.""" return feed_summary(db, user, **params) @router.get("/feed/years") def get_feed_years( params: dict = Depends(_feed_params), user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: """Per-year video counts for the year control: the quick recent-year chips and the "more…" per-year-counts dialog. Every filter EXCEPT the publish-date narrowing applies, so picking a year re-queries the feed but not this list. Response: {"years": [{"year": , "count": }]}, newest year first.""" return {"years": feed_years(db, user, **params)} def _video_or_404(db: Session, video_id: str) -> None: if queries.video(db, video_id) is None: raise not_found("video") class VideoStateIn(BaseModel): status: str @router.post("/videos/{video_id}/state") def set_video_state( video_id: str, body: VideoStateIn, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: status = body.status if status not in VALID_STATES: raise refused(400, "invalid_status") _video_or_404(db, video_id) if status == "new": try: service.unmark(db, user, video_id) db.commit() # silent-ok: the documented lost race — a concurrent player checkpoint already removed or # moved this row (`service.unmark`). The state the caller asked for holds either way. except StaleDataError: db.rollback() return {"ok": True, "video_id": video_id, "status": "new"} service.mark(db, user, video_id, status) db.commit() return {"ok": True, "video_id": video_id, "status": status} @router.delete("/videos/{video_id}/state") def clear_video_state( video_id: str, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: """Reset a video to pristine for this user — drop the whole VideoState row (status, watch position and watched_at), as if it had never been opened. Saved/playlist membership lives elsewhere and is untouched.""" try: service.clear_state(db, user, video_id) db.commit() # silent-ok: the row went under the delete, which is the outcome the delete wanted. except StaleDataError: db.rollback() return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0} @router.post("/videos/{video_id}/progress") def set_video_progress( video_id: str, body: ProgressIn, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: """Checkpoint the in-app player's resume position (called periodically while playing). Stores a per-user position on the video_states row without touching watch status, so a partially-watched video can render a progress bar and match the "in progress" filter. Trivially-early and near-finished positions clear the position rather than store it.""" position = kept_position(body) _video_or_404(db, video_id) if not position: try: service.set_progress(db, user, video_id, 0) db.commit() # silent-ok: a concurrent state change (the end-of-playback "watched" mark) removed or # updated this row already — the lost race; the response stays true. except StaleDataError: db.rollback() return {"ok": True, "video_id": video_id, "position_seconds": 0} service.set_progress(db, user, video_id, position) db.commit() return {"ok": True, "video_id": video_id, "position_seconds": position} @router.get("/videos/{video_id}") def get_video_detail( video_id: str, user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: """On-demand detail (description, like count) — kept out of the feed list payload so the feed stays lean; fetched lazily, e.g. for the title hover popover. Videos we already store are served from the DB for free. A video that isn't in our DB (e.g. a YouTube link inside another video's description that the in-app player navigated to) is resolved via the YouTube API (videos.list, 1 unit, attributed to the requesting user).""" v = queries.video(db, video_id) if v is not None: return { "id": v.id, "description": v.description, "like_count": v.like_count, "in_db": True, "channel_id": v.channel_id, "channel_title": v.channel.title if v.channel else None, "published_at": iso(v.published_at), "view_count": v.view_count, "duration_seconds": v.duration_seconds, } # The DB path above is free; the off-catalog YouTube lookup below spends a quota unit. try: with ( quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP), YouTubeClient(db, user) as yt, ): items = yt.get_videos([video_id]) except YouTubeError as exc: raise youtube_refusal(exc) from exc if not items: raise not_found("video") snippet = items[0].get("snippet", {}) stats = items[0].get("statistics", {}) likes = stats.get("likeCount") views = stats.get("viewCount") return { "id": video_id, "description": snippet.get("description"), "like_count": int(likes) if likes is not None else None, "in_db": False, "channel_id": snippet.get("channelId"), "channel_title": snippet.get("channelTitle"), "published_at": snippet.get("publishedAt"), "view_count": int(views) if views is not None else None, "duration_seconds": parse_iso8601_duration( items[0].get("contentDetails", {}).get("duration") ), } # --- Google Takeout ------------------------------------------------------------------------------ # One upload at a time per user, ten an hour, and two at once across every user: a file's bytes are # held whole while it is read (its text a window at a time), so these bound what the accounts can # make the process hold. The section uploads its files one after another. _takeout_limiter = RateLimiter(max_events=10, window_seconds=3600) _takeout_uploading: set[int] = set() _takeout_uploading_guard = threading.Lock() _TAKEOUT_AT_ONCE = 2 async def _takeout_body( request: Request, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> AsyncIterator[bytes]: # The sign-in check has run, and it was a SELECT: hand the connection back before a file of # megabytes arrives, so a slow upload does not hold one `idle in transaction`. The user's id is # taken first: the release expires the user, and reading it after would reload it, taking a # connection straight back out. uid = user.id release_read_connection(db) with _takeout_uploading_guard: busy = uid in _takeout_uploading full = not busy and len(_takeout_uploading) >= _TAKEOUT_AT_ONCE taken = not busy and not full if taken: _takeout_uploading.add(uid) if not taken or not _takeout_limiter.allow(str(uid)): if taken: with _takeout_uploading_guard: _takeout_uploading.discard(uid) # Read to the end before answering: refused mid-upload, the browser reports a network # failure instead of the refusal (capped_stream_to says why). await drain_body(request) if busy: raise refused(429, "takeout_busy") if full: raise refused(429, "takeout_server_busy") raise refused(429, "takeout_cooldown") try: yield await capped_body(request, takeout.MAX_BYTES) finally: with _takeout_uploading_guard: _takeout_uploading.discard(uid) @router.post("/takeout") def upload_takeout( # The sign-in check FIRST: the body's own dependency needs the signed-in user, so a stranger's # file is refused before a byte of it is read. user: User = Depends(current_user), body: bytes = Depends(_takeout_body), db: Session = Depends(get_db), ): """One file of the user's Google Takeout export — the watch history or the Watch later list. Its videos are kept and the import runs in the background; its outcome reaches the user's inbox.""" try: parsed = takeout.parse(body) except takeout.TakeoutError as exc: if exc.reason == "no_videos": raise refused(422, "takeout_no_videos") from None if exc.reason == "too_many": raise refused(422, "takeout_list_too_long", limit=takeout.WATCH_LATER_MAX) from None raise refused(422, "takeout_unreadable") from None try: takeout.stage(db, user, parsed) except takeout.TooMany: db.rollback() raise refused(422, "takeout_too_many", limit=takeout.ROWS_MAX) from None db.commit() scheduler.run_ad_hoc( scheduler.TAKEOUT_IMPORT, partial(takeout.run_import, user.id), actor_id=user.id, target=takeout.run_target(user.id), notify=True, ) return { "ok": True, "kind": parsed.kind, "videos": len(parsed.videos), "entries": parsed.entries, "ads": parsed.ads, "other": parsed.other, "malformed": parsed.malformed, } @router.get("/takeout") def takeout_status(user: User = Depends(current_user), db: Session = Depends(get_db)): """What the user's Takeout import has done: its last run (none before the first upload), and every video kept from the export, by kind and outcome — a video the catalog did not know yet counts as "waiting" until it does, and is marked then.""" target = takeout.run_target(user.id) run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, target) return { # Queued and not started: its run row is not written yet, and the last run is still the # previous one, so the page waits for it rather than taking that one for this upload's. "queued": scheduler.ad_hoc_queued(scheduler.TAKEOUT_IMPORT, target), "max_bytes": takeout.MAX_BYTES, "run": None if run is None else { "status": run.status, "started_at": iso(run.started_at), "finished_at": iso(run.finished_at), "result": run.result, "failure": run.failure, }, **queries.takeout_summary(db, user), } # --- channels watched but not followed ----------------------------------------------------------- @router.get("/takeout/channels") def watched_channels(user: User = Depends(current_user), db: Session = Depends(get_db)) -> dict: """The channels the user's Takeout history shows them watching often but not following: each with how many of its videos over five minutes they watched, and when last. `pending` is how many more channels may join once the import has asked YouTube for their videos' lengths, so a list still being made does not read as the whole of it; `imported` tells "no history uploaded" from "nothing to suggest".""" return { "imported": queries.has_watch_history(db, user), "pending": queries.watched_channels_pending(db, user), "channels": [ { "id": ch.id, "title": ch.title, "handle": ch.handle, "thumbnail_url": ch.thumbnail_url, "subscriber_count": ch.subscriber_count, "video_count": ch.video_count, "description": ch.description, "watched": int(n), "last_watched_at": iso(last), } for ch, n, last in queries.watched_channels(db, user) ], } @router.post("/takeout/channels/{channel_id}/videos") def watched_channel_videos( channel_id: str, user: User = Depends(current_user), db: Session = Depends(get_db) ) -> dict: """The videos of a channel that the user's list counts, ready to play, last watched first. The channel is explored for the user (as its page does), and the videos the catalog lacks come in as the exploration's: the player needs them in the catalog, the shared Library does not get them, and the cleanup reclaims them once nobody keeps them. `gone`: how many YouTube no longer has.""" channel = catalog_queries.channel(db, channel_id) if channel is None: raise not_found("channel") if catalog_queries.is_blocked(db, user, channel_id): raise refused(403, "channel_blocked") counted, gone = queries.watched_channel_videos(db, user, channel_id) ids = [vid for vid, _ in counted] known = queries.known_video_ids(db, ids) missing = [vid for vid in ids if vid not in known] if missing and quota.backfill_reserve_reached(): raise refused(429, "quota_reserve") catalog_service.explore(db, user, channel) db.commit() if missing: try: with ( quota.attribute(user.id, quota.QuotaAction.CHANNELS_EXPLORE), YouTubeClient(db, user) as yt, ): ensure_videos(db, yt, missing, explore=True) except YouTubeError as exc: raise youtube_refusal(exc) from exc items = queries.feed_rows_by_ids(db, user, ids) watched_at = dict(counted) for item in items: item["takeout_watched_at"] = iso(watched_at[item["id"]]) # A video YouTube still had when its length was asked may have gone since. return {"ok": True, "items": items, "gone": gone + len(ids) - len(items)} @router.put("/takeout/channels/{channel_id}/dismissal") def dismiss_watched_channel( channel_id: str = Path(max_length=32), user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: """Put a channel out of the user's list (it stays out until restored).""" service.dismiss_watched_channel(db, user, channel_id) db.commit() return {"ok": True, "channel_id": channel_id} @router.delete("/takeout/channels/{channel_id}/dismissal") def restore_watched_channel( channel_id: str = Path(max_length=32), user: User = Depends(current_user), db: Session = Depends(get_db), ) -> dict: """Let a channel the user put out of the list back on it.""" service.restore_watched_channel(db, user, channel_id) db.commit() return {"ok": True, "channel_id": channel_id}