peter
5d9952ab1e
A watched video the catalog did not know at import time waits in `takeout_items`. The Shorts job now settles those rows right after its probe (the only place a video's Short-ness becomes known), every user's, a commit each: `reader.takeout.apply_waiting`, its count in the job's result. `GET /api/takeout` is what the settings page reads: the user's last import run (`jobruns.last_run`) and the kept videos by kind and outcome, "waiting" for one not known yet.
369 lines
14 KiB
Python
369 lines
14 KiB
Python
from datetime import date
|
|
from functools import partial
|
|
|
|
from fastapi import APIRouter, Depends, Query, Request
|
|
from pydantic import BaseModel
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.orm.exc import StaleDataError
|
|
|
|
from app import jobruns, quota, scheduler
|
|
from app.auth import current_user
|
|
from app.db import get_db, release_read_connection
|
|
from app.mailtext import lang_for_user
|
|
from app.feedquery import BadCursor, facet_counts, feed_count, feed_page, feed_summary, feed_years
|
|
from app.models import User
|
|
from app.reader import queries, service, takeout
|
|
from app.routes._common import (
|
|
ProgressIn,
|
|
capped_body,
|
|
kept_position,
|
|
not_found,
|
|
youtube_refusal,
|
|
)
|
|
from app.sync.videos import parse_iso8601_duration
|
|
from app.utils import iso
|
|
from app.youtube.client import YouTubeClient, YouTubeError
|
|
from app.refusal import refused
|
|
|
|
router = APIRouter(prefix="/api", tags=["feed"])
|
|
|
|
# "saved" used to be a status; it's now membership in the built-in Watch later playlist.
|
|
VALID_STATES = {"new", "watched", "hidden"}
|
|
|
|
|
|
# Shared query parameters for /feed and /feed/count.
|
|
def _feed_params(
|
|
tags: list[int] = Query(default=[]),
|
|
# Per-category tag match: each of language / topic / other independently ORs (any) or ANDs (all)
|
|
# its own selected tags. Defaults to "or". (Superseded the single global `tag_mode`.)
|
|
tag_mode_language: str = "or",
|
|
tag_mode_topic: str = "or",
|
|
tag_mode_other: str = "or",
|
|
# "No tag" filter: keep only channels the user has not personally tagged (no user-owned ChannelTag).
|
|
untagged: bool = False,
|
|
channel_ids: list[str] = Query(default=[]),
|
|
q: str | None = None,
|
|
min_duration: int | None = None,
|
|
max_duration: int | None = None,
|
|
max_age_days: int | None = None,
|
|
published_after: date | None = None,
|
|
published_before: date | None = None,
|
|
show_normal: bool = True,
|
|
include_shorts: bool = False,
|
|
include_live: bool = False,
|
|
show: str = "unwatched",
|
|
scope: str = "my",
|
|
library_source: str = "organic",
|
|
) -> dict:
|
|
return {
|
|
"tags": tags,
|
|
"tag_modes": {
|
|
"language": tag_mode_language,
|
|
"topic": tag_mode_topic,
|
|
"other": tag_mode_other,
|
|
},
|
|
"untagged": untagged,
|
|
"channel_ids": channel_ids,
|
|
"q": q,
|
|
"min_duration": min_duration,
|
|
"max_duration": max_duration,
|
|
"max_age_days": max_age_days,
|
|
"published_after": published_after,
|
|
"published_before": published_before,
|
|
"show_normal": show_normal,
|
|
"include_shorts": include_shorts,
|
|
"include_live": include_live,
|
|
"show": show,
|
|
"scope": scope,
|
|
"library_source": library_source,
|
|
}
|
|
|
|
|
|
@router.get("/feed")
|
|
def get_feed(
|
|
params: dict = Depends(_feed_params),
|
|
sort: str = "newest",
|
|
seed: int = 0,
|
|
limit: int = Query(default=60, le=200),
|
|
cursor: str | None = None,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
try:
|
|
return feed_page(
|
|
db,
|
|
user,
|
|
params,
|
|
sort=sort,
|
|
seed=seed,
|
|
limit=limit,
|
|
cursor=cursor,
|
|
lang=lang_for_user(user),
|
|
)
|
|
except BadCursor:
|
|
# `from None`: a malformed cursor is an expected bad-input signal, not an error to chain.
|
|
raise refused(400, "invalid_cursor") from None
|
|
|
|
|
|
@router.get("/feed/count")
|
|
def get_feed_count(
|
|
params: dict = Depends(_feed_params),
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
# SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference
|
|
# oracle that test_feed_summary_matches_count_plus_facets asserts /feed/summary's `count` against;
|
|
# remove both together with that test if this endpoint is ever retired.
|
|
return {"count": feed_count(db, user, **params)}
|
|
|
|
|
|
@router.get("/facets")
|
|
def get_facets(
|
|
params: dict = Depends(_feed_params),
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Per-tag channel counts for the *current* filter context, so the sidebar can show
|
|
live counts and drop chips that no longer match anything. Each category is counted with
|
|
every other filter applied (scope, channel, date, content type, search, watch state,
|
|
and the other category's tags) but its own selections ignored — standard drill-down
|
|
faceting, so selecting one topic doesn't zero out the rest of the topics.
|
|
|
|
The count is the number of distinct channels that have at least one video in the current
|
|
view, matching the existing channel-count chip semantics. JSON object keys are strings
|
|
(tag ids).
|
|
|
|
SUPERSEDED by /feed/summary (PERF-13) — the app no longer calls this. Kept as the reference
|
|
oracle for test_feed_summary_matches_count_plus_facets (it asserts /feed/summary's `facets`
|
|
against these `counts`); retire the two together."""
|
|
return {"counts": facet_counts(db, user, **params)}
|
|
|
|
|
|
@router.get("/feed/summary")
|
|
def get_feed_summary(
|
|
params: dict = Depends(_feed_params),
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""The sidebar's total video count + per-tag channel facet counts in ONE request and ONE grouped
|
|
query, replacing the separate /feed/count (1 pass) + /facets (3 passes) the FE fired on every filter
|
|
change (PERF-13). Response: {"count": <videos>, "facets": {<tag_id>: <channel_count>}} — the same
|
|
numbers /feed/count's `count` and /facets' `counts` return, so the two are interchangeable."""
|
|
return feed_summary(db, user, **params)
|
|
|
|
|
|
@router.get("/feed/years")
|
|
def get_feed_years(
|
|
params: dict = Depends(_feed_params),
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Per-year video counts for the year control: the quick recent-year chips and the
|
|
"more…" per-year-counts dialog. Every filter EXCEPT the publish-date narrowing applies, so picking a
|
|
year re-queries the feed but not this list. Response: {"years": [{"year": <int>, "count": <int>}]},
|
|
newest year first."""
|
|
return {"years": feed_years(db, user, **params)}
|
|
|
|
|
|
def _video_or_404(db: Session, video_id: str) -> None:
|
|
if queries.video(db, video_id) is None:
|
|
raise not_found("video")
|
|
|
|
|
|
class VideoStateIn(BaseModel):
|
|
status: str
|
|
|
|
|
|
@router.post("/videos/{video_id}/state")
|
|
def set_video_state(
|
|
video_id: str,
|
|
body: VideoStateIn,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
status = body.status
|
|
if status not in VALID_STATES:
|
|
raise refused(400, "invalid_status")
|
|
_video_or_404(db, video_id)
|
|
if status == "new":
|
|
try:
|
|
service.unmark(db, user, video_id)
|
|
db.commit()
|
|
# silent-ok: the documented lost race — a concurrent player checkpoint already removed or
|
|
# moved this row (`service.unmark`). The state the caller asked for holds either way.
|
|
except StaleDataError:
|
|
db.rollback()
|
|
return {"ok": True, "video_id": video_id, "status": "new"}
|
|
service.mark(db, user, video_id, status)
|
|
db.commit()
|
|
return {"ok": True, "video_id": video_id, "status": status}
|
|
|
|
|
|
@router.delete("/videos/{video_id}/state")
|
|
def clear_video_state(
|
|
video_id: str,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Reset a video to pristine for this user — drop the whole VideoState row (status,
|
|
watch position and watched_at), as if it had never been opened. Saved/playlist membership
|
|
lives elsewhere and is untouched."""
|
|
try:
|
|
service.clear_state(db, user, video_id)
|
|
db.commit()
|
|
# silent-ok: the row went under the delete, which is the outcome the delete wanted.
|
|
except StaleDataError:
|
|
db.rollback()
|
|
return {"ok": True, "video_id": video_id, "status": "new", "position_seconds": 0}
|
|
|
|
|
|
@router.post("/videos/{video_id}/progress")
|
|
def set_video_progress(
|
|
video_id: str,
|
|
body: ProgressIn,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""Checkpoint the in-app player's resume position (called periodically while playing).
|
|
Stores a per-user position on the video_states row without touching watch status, so a
|
|
partially-watched video can render a progress bar and match the "in progress" filter.
|
|
Trivially-early and near-finished positions clear the position rather than store it."""
|
|
position = kept_position(body)
|
|
_video_or_404(db, video_id)
|
|
if not position:
|
|
try:
|
|
service.set_progress(db, user, video_id, 0)
|
|
db.commit()
|
|
# silent-ok: a concurrent state change (the end-of-playback "watched" mark) removed or
|
|
# updated this row already — the lost race; the response stays true.
|
|
except StaleDataError:
|
|
db.rollback()
|
|
return {"ok": True, "video_id": video_id, "position_seconds": 0}
|
|
service.set_progress(db, user, video_id, position)
|
|
db.commit()
|
|
return {"ok": True, "video_id": video_id, "position_seconds": position}
|
|
|
|
|
|
@router.get("/videos/{video_id}")
|
|
def get_video_detail(
|
|
video_id: str,
|
|
user: User = Depends(current_user),
|
|
db: Session = Depends(get_db),
|
|
) -> dict:
|
|
"""On-demand detail (description, like count) — kept out of the feed list payload
|
|
so the feed stays lean; fetched lazily, e.g. for the title hover popover.
|
|
|
|
Videos we already store are served from the DB for free. A video that isn't in
|
|
our DB (e.g. a YouTube link inside another video's description that the in-app
|
|
player navigated to) is resolved via the YouTube API (videos.list, 1 unit,
|
|
attributed to the requesting user)."""
|
|
v = queries.video(db, video_id)
|
|
if v is not None:
|
|
return {
|
|
"id": v.id,
|
|
"description": v.description,
|
|
"like_count": v.like_count,
|
|
"in_db": True,
|
|
"channel_id": v.channel_id,
|
|
"channel_title": v.channel.title if v.channel else None,
|
|
"published_at": iso(v.published_at),
|
|
"view_count": v.view_count,
|
|
"duration_seconds": v.duration_seconds,
|
|
}
|
|
|
|
# The DB path above is free; the off-catalog YouTube lookup below spends a quota unit.
|
|
try:
|
|
with (
|
|
quota.attribute(user.id, quota.QuotaAction.VIDEOS_LOOKUP),
|
|
YouTubeClient(db, user) as yt,
|
|
):
|
|
items = yt.get_videos([video_id])
|
|
except YouTubeError as exc:
|
|
raise youtube_refusal(exc) from exc
|
|
if not items:
|
|
raise not_found("video")
|
|
|
|
snippet = items[0].get("snippet", {})
|
|
stats = items[0].get("statistics", {})
|
|
likes = stats.get("likeCount")
|
|
views = stats.get("viewCount")
|
|
return {
|
|
"id": video_id,
|
|
"description": snippet.get("description"),
|
|
"like_count": int(likes) if likes is not None else None,
|
|
"in_db": False,
|
|
"channel_id": snippet.get("channelId"),
|
|
"channel_title": snippet.get("channelTitle"),
|
|
"published_at": snippet.get("publishedAt"),
|
|
"view_count": int(views) if views is not None else None,
|
|
"duration_seconds": parse_iso8601_duration(
|
|
items[0].get("contentDetails", {}).get("duration")
|
|
),
|
|
}
|
|
|
|
|
|
# --- Google Takeout ------------------------------------------------------------------------------
|
|
|
|
|
|
async def _takeout_body(request: Request, db: Session = Depends(get_db)) -> bytes:
|
|
# The sign-in check has run, and it was a SELECT: hand the connection back before a file of
|
|
# megabytes arrives, so a slow upload does not hold one `idle in transaction`.
|
|
release_read_connection(db)
|
|
return await capped_body(request, takeout.MAX_BYTES)
|
|
|
|
|
|
@router.post("/takeout")
|
|
def upload_takeout(
|
|
# The sign-in check FIRST: dependencies resolve in order, so a stranger's file is refused before
|
|
# a byte of it is read.
|
|
user: User = Depends(current_user),
|
|
body: bytes = Depends(_takeout_body),
|
|
db: Session = Depends(get_db),
|
|
):
|
|
"""One file of the user's Google Takeout export — the watch history or the Watch later list.
|
|
Its videos are kept and the import runs in the background; its outcome reaches the user's inbox."""
|
|
try:
|
|
parsed = takeout.parse(body)
|
|
except takeout.TakeoutError as exc:
|
|
if exc.reason == "no_videos":
|
|
raise refused(422, "takeout_no_videos") from None
|
|
raise refused(422, "takeout_unreadable") from None
|
|
service.stage_takeout(db, user, parsed.kind, parsed.videos)
|
|
db.commit()
|
|
scheduler.run_ad_hoc(
|
|
scheduler.TAKEOUT_IMPORT,
|
|
partial(takeout.run_import, user.id),
|
|
actor_id=user.id,
|
|
target=takeout.run_target(user.id),
|
|
notify=True,
|
|
)
|
|
return {
|
|
"ok": True,
|
|
"kind": parsed.kind,
|
|
"videos": len(parsed.videos),
|
|
"entries": parsed.entries,
|
|
"ads": parsed.ads,
|
|
"other": parsed.other,
|
|
"malformed": parsed.malformed,
|
|
}
|
|
|
|
|
|
@router.get("/takeout")
|
|
def takeout_status(user: User = Depends(current_user), db: Session = Depends(get_db)):
|
|
"""What the user's Takeout import has done: its last run (none before the first upload), and
|
|
every video kept from the export, by kind and outcome — a video the catalog did not know yet
|
|
counts as "waiting" until it does, and is marked then."""
|
|
run = jobruns.last_run(db, scheduler.TAKEOUT_IMPORT, takeout.run_target(user.id))
|
|
return {
|
|
"run": None
|
|
if run is None
|
|
else {
|
|
"status": run.status,
|
|
"started_at": iso(run.started_at),
|
|
"finished_at": iso(run.finished_at),
|
|
"result": run.result,
|
|
"failure": run.failure,
|
|
},
|
|
**queries.takeout_summary(db, user),
|
|
}
|